## Coverage Report

#### Created: 2025-11-22 06:00

next uncovered line (L), next uncovered region (R), next uncovered branch (B)

    /Users/andrewlamb/Software/datafusion/datafusion/common/src/scalar/mod.rs

    Line
    Count
    Source
    1
    // Licensed to the Apache Software Foundation (ASF) under one
    2
    // or more contributor license agreements.  See the NOTICE file
    3
    // distributed with this work for additional information
    4
    // regarding copyright ownership.  The ASF licenses this file
    5
    // to you under the Apache License, Version 2.0 (the
    6
    // "License"); you may not use this file except in compliance
    7
    // with the License.  You may obtain a copy of the License at
    8
    //
    9
    //   http://www.apache.org/licenses/LICENSE-2.0
    10
    //
    11
    // Unless required by applicable law or agreed to in writing,
    12
    // software distributed under the License is distributed on an
    13
    // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
    14
    // KIND, either express or implied.  See the License for the
    15
    // specific language governing permissions and limitations
    16
    // under the License.
    17
    18
    //! [`ScalarValue`]: stores single  values
    19
    20
    mod cache;
    21
    mod consts;
    22
    mod struct_builder;
    23
    24
    use std::borrow::Borrow;
    25
    use std::cmp::Ordering;
    26
    use std::collections::{HashSet, VecDeque};
    27
    use std::convert::Infallible;
    28
    use std::fmt;
    29
    use std::hash::Hash;
    30
    use std::hash::Hasher;
    31
    use std::iter::repeat_n;
    32
    use std::mem::{size_of, size_of_val};
    33
    use std::str::FromStr;
    34
    use std::sync::Arc;
    35
    36
    use crate::assert_or_internal_err;
    37
    use crate::cast::{
    38
        as_binary_array, as_binary_view_array, as_boolean_array, as_date32_array,
    39
        as_date64_array, as_decimal128_array, as_decimal256_array, as_decimal32_array,
    40
        as_decimal64_array, as_dictionary_array, as_duration_microsecond_array,
    41
        as_duration_millisecond_array, as_duration_nanosecond_array,
    42
        as_duration_second_array, as_fixed_size_binary_array, as_fixed_size_list_array,
    43
        as_float16_array, as_float32_array, as_float64_array, as_int16_array, as_int32_array,
    44
        as_int64_array, as_int8_array, as_interval_dt_array, as_interval_mdn_array,
    45
        as_interval_ym_array, as_large_binary_array, as_large_list_array,
    46
        as_large_string_array, as_string_array, as_string_view_array,
    47
        as_time32_millisecond_array, as_time32_second_array, as_time64_microsecond_array,
    48
        as_time64_nanosecond_array, as_timestamp_microsecond_array,
    49
        as_timestamp_millisecond_array, as_timestamp_nanosecond_array,
    50
        as_timestamp_second_array, as_uint16_array, as_uint32_array, as_uint64_array,
    51
        as_uint8_array, as_union_array,
    52
    };
    53
    use crate::error::{DataFusionError, Result, _exec_err, _internal_err, _not_impl_err};
    54
    use crate::format::DEFAULT_CAST_OPTIONS;
    55
    use crate::hash_utils::create_hashes;
    56
    use crate::utils::SingleRowListArrayBuilder;
    57
    use crate::{_internal_datafusion_err, arrow_datafusion_err};
    58
    use arrow::array::{
    59
        new_empty_array, new_null_array, Array, ArrayData, ArrayRef, ArrowNativeTypeOp,
    60
        ArrowPrimitiveType, AsArray, BinaryArray, BinaryViewArray, BooleanArray, Date32Array,
    61
        Date64Array, Decimal128Array, Decimal256Array, Decimal32Array, Decimal64Array,
    62
        DictionaryArray, DurationMicrosecondArray, DurationMillisecondArray,
    63
        DurationNanosecondArray, DurationSecondArray, FixedSizeBinaryArray,
    64
        FixedSizeListArray, Float16Array, Float32Array, Float64Array, GenericListArray,
    65
        Int16Array, Int32Array, Int64Array, Int8Array, IntervalDayTimeArray,
    66
        IntervalMonthDayNanoArray, IntervalYearMonthArray, LargeBinaryArray, LargeListArray,
    67
        LargeStringArray, ListArray, MapArray, MutableArrayData, OffsetSizeTrait,
    68
        PrimitiveArray, Scalar, StringArray, StringViewArray, StructArray,
    69
        Time32MillisecondArray, Time32SecondArray, Time64MicrosecondArray,
    70
        Time64NanosecondArray, TimestampMicrosecondArray, TimestampMillisecondArray,
    71
        TimestampNanosecondArray, TimestampSecondArray, UInt16Array, UInt32Array,
    72
        UInt64Array, UInt8Array, UnionArray,
    73
    };
    74
    use arrow::buffer::{BooleanBuffer, ScalarBuffer};
    75
    use arrow::compute::kernels::cast::{cast_with_options, CastOptions};
    76
    use arrow::compute::kernels::numeric::{
    77
        add, add_wrapping, div, mul, mul_wrapping, rem, sub, sub_wrapping,
    78
    };
    79
    use arrow::datatypes::{
    80
        i256, validate_decimal_precision_and_scale, ArrowDictionaryKeyType, ArrowNativeType,
    81
        ArrowTimestampType, DataType, Date32Type, Decimal128Type, Decimal256Type,
    82
        Decimal32Type, Decimal64Type, DecimalType, Field, Float32Type, Int16Type, Int32Type,
    83
        Int64Type, Int8Type, IntervalDayTime, IntervalDayTimeType, IntervalMonthDayNano,
    84
        IntervalMonthDayNanoType, IntervalUnit, IntervalYearMonthType, TimeUnit,
    85
        TimestampMicrosecondType, TimestampMillisecondType, TimestampNanosecondType,
    86
        TimestampSecondType, UInt16Type, UInt32Type, UInt64Type, UInt8Type, UnionFields,
    87
        UnionMode, DECIMAL128_MAX_PRECISION,
    88
    };
    89
    use arrow::util::display::{array_value_to_string, ArrayFormatter, FormatOptions};
    90
    use cache::{get_or_create_cached_key_array, get_or_create_cached_null_array};
    91
    use chrono::{Duration, NaiveDate};
    92
    use half::f16;
    93
    pub use struct_builder::ScalarStructBuilder;
    94
    95
    const SECONDS_PER_DAY: i64 = 86_400;
    96
    const MILLIS_PER_DAY: i64 = SECONDS_PER_DAY * 1_000;
    97
    const MICROS_PER_DAY: i64 = MILLIS_PER_DAY * 1_000;
    98
    const NANOS_PER_DAY: i64 = MICROS_PER_DAY * 1_000;
    99
    const MICROS_PER_MILLISECOND: i64 = 1_000;
    100
    const NANOS_PER_MILLISECOND: i64 = 1_000_000;
    101
    102
    /// Returns the multiplier that converts the input date representation into the
    103
    /// desired timestamp unit, if the conversion requires a multiplication that can
    104
    /// overflow an `i64`.
    105
    493
    pub fn date_to_timestamp_multiplier(
    106
    493
        source_type: &DataType,
    107
    493
        target_type: &DataType,
    108
    493
    ) -> Option<i64> {
    109
    493
        let DataType::Timestamp(target_unit0, _) = target_type else {
    110
    493
            return None;
    111
        };
    112
    113
        // Only `Timestamp` target types have a time unit; otherwise no
    114
        // multiplier applies (handled above). The function returns `Some(m)`
    115
        // when converting the `source_type` to `target_type` requires a
    116
        // multiplication that could overflow `i64`. It returns `None` when
    117
        // the conversion is a division or otherwise doesn't require a
    118
        // multiplication (e.g. Date64 -> Second).
    119
    0
        match source_type {
    120
            // Date32 stores days since epoch. Converting to any timestamp
    121
            // unit requires multiplying by the per-day factor (seconds,
    122
            // milliseconds, microseconds, nanoseconds).
    123
    0
            DataType::Date32 => Some(match target_unit {
    124
    0
                TimeUnit::Second => SECONDS_PER_DAY,
    125
    0
                TimeUnit::Millisecond => MILLIS_PER_DAY,
    126
    0
                TimeUnit::Microsecond => MICROS_PER_DAY,
    127
    0
                TimeUnit::Nanosecond => NANOS_PER_DAY,
    128
            }),
    129
    130
            // Date64 stores milliseconds since epoch. Converting to
    131
            // seconds is a division (no multiplication), so return `None`.
    132
            // Converting to milliseconds is 1:1 (multiplier 1). Converting
    133
            // to micro/nano requires multiplying by 1_000 / 1_000_000.
    134
    0
            DataType::Date64 => match target_unit {
    135
    0
                TimeUnit::Second => None,
    136
                // Converting Date64 (ms since epoch) to millisecond timestamps
    137
                // is an identity conversion and does not require multiplication.
    138
                // Returning `None` indicates no multiplication-based overflow
    139
                // check is necessary.
    140
    0
                TimeUnit::Millisecond => None,
    141
    0
                TimeUnit::Microsecond => Some(MICROS_PER_MILLISECOND),
    142
    0
                TimeUnit::Nanosecond => Some(NANOS_PER_MILLISECOND),
    143
            },
    144
    145
    0
            _ => None,
    146
        }
    147
    493
    }
    148
    149
    /// Ensures the provided value can be represented as a timestamp with the given
    150
    /// multiplier. Returns an [`DataFusionError::Execution`] when the converted
    151
    /// value would overflow the timestamp range.
    152
    0
    pub fn ensure_timestamp_in_bounds(
    153
    0
        value: i64,
    154
    0
        multiplier: i64,
    155
    0
        source_type: &DataType,
    156
    0
        target_type: &DataType,
    157
    0
    ) -> Result<()> {
    158
    0
        if multiplier <= 1 {
    159
    0
            return Ok(());
    160
    0
        }
    161
    162
    0
        if value.checked_mul(multiplier).is_none() {
    163
    0
            let target = format_timestamp_type_for_error(target_type);
    164
    0
            _exec_err!(
    165
    0
                "Cannot cast {} value {} to {}: converted value exceeds the representable i64 range",
    166
                source_type,
    167
                value,
    168
                target
    169
            )
    170
        } else {
    171
    0
            Ok(())
    172
        }
    173
    0
    }
    174
    175
    /// Format a `DataType::Timestamp` into a short, stable string used in
    176
    /// user-facing error messages.
    177
    0
    pub(crate) fn format_timestamp_type_for_error(target_type: &DataType) -> String {
    178
    0
        match target_type {
    179
    0
            DataType::Timestamp(unit, _) => {
    180
    0
                let s = match unit {
    181
    0
                    TimeUnit::Second => "s",
    182
    0
                    TimeUnit::Millisecond => "ms",
    183
    0
                    TimeUnit::Microsecond => "us",
    184
    0
                    TimeUnit::Nanosecond => "ns",
    185
                };
    186
    0
                format!("Timestamp({s})")
    187
            }
    188
    0
            other => format!("{other}"),
    189
        }
    190
    0
    }
    191
    192
    /// A dynamically typed, nullable single value.
    193
    ///
    194
    /// While an arrow  [`Array`]) stores one or more values of the same type, in a
    195
    /// single column, a `ScalarValue` stores a single value of a single type, the
    196
    /// equivalent of 1 row and one column.
    197
    ///
    198
    /// ```text
    199
    ///  ┌────────┐
    200
    ///  │ value1 │
    201
    ///  │ value2 │                  ┌────────┐
    202
    ///  │ value3 │                  │ value2 │
    203
    ///  │  ...   │                  └────────┘
    204
    ///  │ valueN │
    205
    ///  └────────┘
    206
    ///
    207
    ///    Array                     ScalarValue
    208
    ///
    209
    /// stores multiple,             stores a single,
    210
    /// possibly null, values of     possible null, value
    211
    /// the same type
    212
    /// ```
    213
    ///
    214
    /// # Performance
    215
    ///
    216
    /// In general, performance will be better using arrow [`Array`]s rather than
    217
    /// [`ScalarValue`], as it is far more efficient to process multiple values at
    218
    /// once (vectorized processing).
    219
    ///
    220
    /// # Example
    221
    /// ```
    222
    /// # use datafusion_common::ScalarValue;
    223
    /// // Create single scalar value for an Int32 value
    224
    /// let s1 = ScalarValue::Int32(Some(10));
    225
    ///
    226
    /// // You can also create values using the From impl:
    227
    /// let s2 = ScalarValue::from(10i32);
    228
    /// assert_eq!(s1, s2);
    229
    /// ```
    230
    ///
    231
    /// # Null Handling
    232
    ///
    233
    /// `ScalarValue` represents null values in the same way as Arrow. Nulls are
    234
    /// "typed" in the sense that a null value in an [`Int32Array`] is different
    235
    /// from a null value in a [`Float64Array`], and is different from the values in
    236
    /// a [`NullArray`].
    237
    ///
    238
    /// ```
    239
    /// # fn main() -> datafusion_common::Result<()> {
    240
    /// # use std::collections::hash_set::Difference;
    241
    /// # use datafusion_common::ScalarValue;
    242
    /// # use arrow::datatypes::DataType;
    243
    /// // You can create a 'null' Int32 value directly:
    244
    /// let s1 = ScalarValue::Int32(None);
    245
    ///
    246
    /// // You can also create a null value for a given datatype:
    247
    /// let s2 = ScalarValue::try_from(&DataType::Int32)?;
    248
    /// assert_eq!(s1, s2);
    249
    ///
    250
    /// // Note that this is DIFFERENT than a `ScalarValue::Null`
    251
    /// let s3 = ScalarValue::Null;
    252
    /// assert_ne!(s1, s3);
    253
    /// # Ok(())
    254
    /// # }
    255
    /// ```
    256
    ///
    257
    /// # Nested Types
    258
    ///
    259
    /// `List` / `LargeList` / `FixedSizeList` / `Struct` / `Map` are represented as a
    260
    /// single element array of the corresponding type.
    261
    ///
    262
    /// ## Example: Creating [`ScalarValue::Struct`] using [`ScalarStructBuilder`]
    263
    /// ```
    264
    /// # use std::sync::Arc;
    265
    /// # use arrow::datatypes::{DataType, Field};
    266
    /// # use datafusion_common::{ScalarValue, scalar::ScalarStructBuilder};
    267
    /// // Build a struct like: {a: 1, b: "foo"}
    268
    /// let field_a = Field::new("a", DataType::Int32, false);
    269
    /// let field_b = Field::new("b", DataType::Utf8, false);
    270
    ///
    271
    /// let s1 = ScalarStructBuilder::new()
    272
    ///     .with_scalar(field_a, ScalarValue::from(1i32))
    273
    ///     .with_scalar(field_b, ScalarValue::from("foo"))
    274
    ///     .build();
    275
    /// ```
    276
    ///
    277
    /// ## Example: Creating a null [`ScalarValue::Struct`] using [`ScalarStructBuilder`]
    278
    /// ```
    279
    /// # use std::sync::Arc;
    280
    /// # use arrow::datatypes::{DataType, Field};
    281
    /// # use datafusion_common::{ScalarValue, scalar::ScalarStructBuilder};
    282
    /// // Build a struct representing a NULL value
    283
    /// let fields = vec![
    284
    ///     Field::new("a", DataType::Int32, false),
    285
    ///     Field::new("b", DataType::Utf8, false),
    286
    /// ];
    287
    ///
    288
    /// let s1 = ScalarStructBuilder::new_null(fields);
    289
    /// ```
    290
    ///
    291
    /// ## Example: Creating [`ScalarValue::Struct`] directly
    292
    /// ```
    293
    /// # use std::sync::Arc;
    294
    /// # use arrow::datatypes::{DataType, Field, Fields};
    295
    /// # use arrow::array::{ArrayRef, Int32Array, StructArray, StringArray};
    296
    /// # use datafusion_common::ScalarValue;
    297
    /// // Build a struct like: {a: 1, b: "foo"}
    298
    /// // Field description
    299
    /// let fields = Fields::from(vec![
    300
    ///     Field::new("a", DataType::Int32, false),
    301
    ///     Field::new("b", DataType::Utf8, false),
    302
    /// ]);
    303
    /// // one row arrays for each field
    304
    /// let arrays: Vec<ArrayRef> = vec![
    305
    ///     Arc::new(Int32Array::from(vec![1])),
    306
    ///     Arc::new(StringArray::from(vec!["foo"])),
    307
    /// ];
    308
    /// // no nulls for this array
    309
    /// let nulls = None;
    310
    /// let arr = StructArray::new(fields, arrays, nulls);
    311
    ///
    312
    /// // Create a ScalarValue::Struct directly
    313
    /// let s1 = ScalarValue::Struct(Arc::new(arr));
    314
    /// ```
    315
    ///
    316
    ///
    317
    /// # Further Reading
    318
    /// See [datatypes](https://arrow.apache.org/docs/python/api/datatypes.html) for
    319
    /// details on datatypes and the [format](https://github.com/apache/arrow/blob/master/format/Schema.fbs#L354-L375)
    320
    /// for the definitive reference.
    321
    ///
    322
    /// [`NullArray`]: arrow::array::NullArray
    323
    #[derive(Clone)]
    324
    pub enum ScalarValue {
    325
        /// represents `DataType::Null` (castable to/from any other type)
    326
        Null,
    327
        /// true or false value
    328
        Boolean(Option<bool>),
    329
        /// 16bit float
    330
        Float16(Option<f16>),
    331
        /// 32bit float
    332
        Float32(Option<f32>),
    333
        /// 64bit float
    334
        Float64(Option<f64>),
    335
        /// 32bit decimal, using the i32 to represent the decimal, precision scale
    336
        Decimal32(Option<i32>, u8, i8),
    337
        /// 64bit decimal, using the i64 to represent the decimal, precision scale
    338
        Decimal64(Option<i64>, u8, i8),
    339
        /// 128bit decimal, using the i128 to represent the decimal, precision scale
    340
        Decimal128(Option<i128>, u8, i8),
    341
        /// 256bit decimal, using the i256 to represent the decimal, precision scale
    342
        Decimal256(Option<i256>, u8, i8),
    343
        /// signed 8bit int
    344
        Int8(Option<i8>),
    345
        /// signed 16bit int
    346
        Int16(Option<i16>),
    347
        /// signed 32bit int
    348
        Int32(Option<i32>),
    349
        /// signed 64bit int
    350
        Int64(Option<i64>),
    351
        /// unsigned 8bit int
    352
        UInt8(Option<u8>),
    353
        /// unsigned 16bit int
    354
        UInt16(Option<u16>),
    355
        /// unsigned 32bit int
    356
        UInt32(Option<u32>),
    357
        /// unsigned 64bit int
    358
        UInt64(Option<u64>),
    359
        /// utf-8 encoded string.
    360
        Utf8(Option<String>),
    361
        /// utf-8 encoded string but from view types.
    362
        Utf8View(Option<String>),
    363
        /// utf-8 encoded string representing a LargeString's arrow type.
    364
        LargeUtf8(Option<String>),
    365
        /// binary
    366
        Binary(Option<Vec<u8>>),
    367
        /// binary but from view types.
    368
        BinaryView(Option<Vec<u8>>),
    369
        /// fixed size binary
    370
        FixedSizeBinary(i32, Option<Vec<u8>>),
    371
        /// large binary
    372
        LargeBinary(Option<Vec<u8>>),
    373
        /// Fixed size list scalar.
    374
        ///
    375
        /// The array must be a FixedSizeListArray with length 1.
    376
        FixedSizeList(Arc<FixedSizeListArray>),
    377
        /// Represents a single element of a [`ListArray`] as an [`ArrayRef`]
    378
        ///
    379
        /// The array must be a ListArray with length 1.
    380
        List(Arc<ListArray>),
    381
        /// The array must be a LargeListArray with length 1.
    382
        LargeList(Arc<LargeListArray>),
    383
        /// Represents a single element [`StructArray`] as an [`ArrayRef`]. See
    384
        /// [`ScalarValue`] for examples of how to create instances of this type.
    385
        Struct(Arc<StructArray>),
    386
        /// Represents a single element [`MapArray`] as an [`ArrayRef`].
    387
        Map(Arc<MapArray>),
    388
        /// Date stored as a signed 32bit int days since UNIX epoch 1970-01-01
    389
        Date32(Option<i32>),
    390
        /// Date stored as a signed 64bit int milliseconds since UNIX epoch 1970-01-01
    391
        Date64(Option<i64>),
    392
        /// Time stored as a signed 32bit int as seconds since midnight
    393
        Time32Second(Option<i32>),
    394
        /// Time stored as a signed 32bit int as milliseconds since midnight
    395
        Time32Millisecond(Option<i32>),
    396
        /// Time stored as a signed 64bit int as microseconds since midnight
    397
        Time64Microsecond(Option<i64>),
    398
        /// Time stored as a signed 64bit int as nanoseconds since midnight
    399
        Time64Nanosecond(Option<i64>),
    400
        /// Timestamp Second
    401
        TimestampSecond(Option<i64>, Option<Arc<str>>),
    402
        /// Timestamp Milliseconds
    403
        TimestampMillisecond(Option<i64>, Option<Arc<str>>),
    404
        /// Timestamp Microseconds
    405
        TimestampMicrosecond(Option<i64>, Option<Arc<str>>),
    406
        /// Timestamp Nanoseconds
    407
        TimestampNanosecond(Option<i64>, Option<Arc<str>>),
    408
        /// Number of elapsed whole months
    409
        IntervalYearMonth(Option<i32>),
    410
        /// Number of elapsed days and milliseconds (no leap seconds)
    411
        /// stored as 2 contiguous 32-bit signed integers
    412
        IntervalDayTime(Option<IntervalDayTime>),
    413
        /// A triple of the number of elapsed months, days, and nanoseconds.
    414
        /// Months and days are encoded as 32-bit signed integers.
    415
        /// Nanoseconds is encoded as a 64-bit signed integer (no leap seconds).
    416
        IntervalMonthDayNano(Option<IntervalMonthDayNano>),
    417
        /// Duration in seconds
    418
        DurationSecond(Option<i64>),
    419
        /// Duration in milliseconds
    420
        DurationMillisecond(Option<i64>),
    421
        /// Duration in microseconds
    422
        DurationMicrosecond(Option<i64>),
    423
        /// Duration in nanoseconds
    424
        DurationNanosecond(Option<i64>),
    425
        /// A nested datatype that can represent slots of differing types. Components:
    426
        /// `.0`: a tuple of union `type_id` and the single value held by this Scalar
    427
        /// `.1`: the list of fields, zero-to-one of which will by set in `.0`
    428
        /// `.2`: the physical storage of the source/destination UnionArray from which this Scalar came
    429
        Union(Option<(i8, Box<ScalarValue>)>, UnionFields, UnionMode),
    430
        /// Dictionary type: index type and value
    431
        Dictionary(Box<DataType>, Box<ScalarValue>),
    432
    }
    433
    434
    impl Hash for Fl<f16> {
    435
    0
        fn hash<H: Hasher>(&self, state: &mut H) {
    436
    0
            self.0.to_bits().hash(state);
    437
    0
        }
    438
    }
    439
    440
    // manual implementation of `PartialEq`
    441
    impl PartialEq for ScalarValue {
    442
    2
        fn eq(&self, other: &Self) -> bool {
    443
            use ScalarValue::*;
    444
            // This purposely doesn't have a catch-all "(_, _)" so that
    445
            // any newly added enum variant will require editing this list
    446
            // or else face a compile error
    447
    2
            match (self, other) {
    448
    0
                (Decimal32(v1, p1, s1), Decimal32(v2, p2, s2)) => {
    449
    0
                    v1.eq(v2) && p1.eq(p2) && s1.eq(s2)
    450
                }
    451
    0
                (Decimal32(_, _, _), _) => false,
    452
    0
                (Decimal64(v1, p1, s1), Decimal64(v2, p2, s2)) => {
    453
    0
                    v1.eq(v2) && p1.eq(p2) && s1.eq(s2)
    454
                }
    455
    0
                (Decimal64(_, _, _), _) => false,
    456
    0
                (Decimal128(v1, p1, s1), Decimal128(v2, p2, s2)) => {
    457
    0
                    v1.eq(v2) && p1.eq(p2) && s1.eq(s2)
    458
                }
    459
    0
                (Decimal128(_, _, _), _) => false,
    460
    0
                (Decimal256(v1, p1, s1), Decimal256(v2, p2, s2)) => {
    461
    0
                    v1.eq(v2) && p1.eq(p2) && s1.eq(s2)
    462
                }
    463
    0
                (Decimal256(_, _, _), _) => false,
    464
    0
                (Boolean(v1), Boolean(v2)) => v1.eq(v2),
    465
    0
                (Boolean(_), _) => false,
    466
    0
                (Float32(v1), Float32(v2)) => match (v1, v2) {
    467
    0
                    (Some(f1), Some(f2)) => f1.to_bits() == f2.to_bits(),
    468
    0
                    _ => v1.eq(v2),
    469
                },
    470
    0
                (Float16(v1), Float16(v2)) => match (v1, v2) {
    471
    0
                    (Some(f1), Some(f2)) => f1.to_bits() == f2.to_bits(),
    472
    0
                    _ => v1.eq(v2),
    473
                },
    474
    0
                (Float32(_), _) => false,
    475
    0
                (Float16(_), _) => false,
    476
    0
                (Float64(v1), Float64(v2)) => match (v1, v2) {
    477
    0
                    (Some(f1), Some(f2)) => f1.to_bits() == f2.to_bits(),
    478
    0
                    _ => v1.eq(v2),
    479
                },
    480
    0
                (Float64(_), _) => false,
    481
    0
                (Int8(v1), Int8(v2)) => v1.eq(v2),
    482
    0
                (Int8(_), _) => false,
    483
    0
                (Int16(v1), Int16(v2)) => v1.eq(v2),
    484
    0
                (Int16(_), _) => false,
    485
    0
                (Int32(v1), Int32(v2)) => v1.eq(v2),
    486
    0
                (Int32(_), _) => false,
    487
    0
                (Int64(v1), Int64(v2)) => v1.eq(v2),
    488
    0
                (Int64(_), _) => false,
    489
    0
                (UInt8(v1), UInt8(v2)) => v1.eq(v2),
    490
    0
                (UInt8(_), _) => false,
    491
    0
                (UInt16(v1), UInt16(v2)) => v1.eq(v2),
    492
    0
                (UInt16(_), _) => false,
    493
    0
                (UInt32(v1), UInt32(v2)) => v1.eq(v2),
    494
    0
                (UInt32(_), _) => false,
    495
    0
                (UInt64(v1), UInt64(v2)) => v1.eq(v2),
    496
    0
                (UInt64(_), _) => false,
    497
    1
                (Utf8(v1), Utf8(v2)) => v1.eq(v2),
    498
    0
                (Utf8(_), _) => false,
    499
    0
                (Utf8View(v1), Utf8View(v2)) => v1.eq(v2),
    500
    0
                (Utf8View(_), _) => false,
    501
    0
                (LargeUtf8(v1), LargeUtf8(v2)) => v1.eq(v2),
    502
    0
                (LargeUtf8(_), _) => false,
    503
    0
                (Binary(v1), Binary(v2)) => v1.eq(v2),
    504
    0
                (Binary(_), _) => false,
    505
    0
                (BinaryView(v1), BinaryView(v2)) => v1.eq(v2),
    506
    0
                (BinaryView(_), _) => false,
    507
    0
                (FixedSizeBinary(_, v1), FixedSizeBinary(_, v2)) => v1.eq(v2),
    508
    0
                (FixedSizeBinary(_, _), _) => false,
    509
    0
                (LargeBinary(v1), LargeBinary(v2)) => v1.eq(v2),
    510
    0
                (LargeBinary(_), _) => false,
    511
    0
                (FixedSizeList(v1), FixedSizeList(v2)) => v1.eq(v2),
    512
    0
                (FixedSizeList(_), _) => false,
    513
    0
                (List(v1), List(v2)) => v1.eq(v2),
    514
    0
                (List(_), _) => false,
    515
    0
                (LargeList(v1), LargeList(v2)) => v1.eq(v2),
    516
    0
                (LargeList(_), _) => false,
    517
    0
                (Struct(v1), Struct(v2)) => v1.eq(v2),
    518
    0
                (Struct(_), _) => false,
    519
    0
                (Map(v1), Map(v2)) => v1.eq(v2),
    520
    0
                (Map(_), _) => false,
    521
    0
                (Date32(v1), Date32(v2)) => v1.eq(v2),
    522
    0
                (Date32(_), _) => false,
    523
    0
                (Date64(v1), Date64(v2)) => v1.eq(v2),
    524
    0
                (Date64(_), _) => false,
    525
    0
                (Time32Second(v1), Time32Second(v2)) => v1.eq(v2),
    526
    0
                (Time32Second(_), _) => false,
    527
    0
                (Time32Millisecond(v1), Time32Millisecond(v2)) => v1.eq(v2),
    528
    0
                (Time32Millisecond(_), _) => false,
    529
    0
                (Time64Microsecond(v1), Time64Microsecond(v2)) => v1.eq(v2),
    530
    0
                (Time64Microsecond(_), _) => false,
    531
    0
                (Time64Nanosecond(v1), Time64Nanosecond(v2)) => v1.eq(v2),
    532
    0
                (Time64Nanosecond(_), _) => false,
    533
    0
                (TimestampSecond(v1, _), TimestampSecond(v2, _)) => v1.eq(v2),
    534
    0
                (TimestampSecond(_, _), _) => false,
    535
    0
                (TimestampMillisecond(v1, _), TimestampMillisecond(v2, _)) => v1.eq(v2),
    536
    0
                (TimestampMillisecond(_, _), _) => false,
    537
    0
                (TimestampMicrosecond(v1, _), TimestampMicrosecond(v2, _)) => v1.eq(v2),
    538
    0
                (TimestampMicrosecond(_, _), _) => false,
    539
    0
                (TimestampNanosecond(v1, _), TimestampNanosecond(v2, _)) => v1.eq(v2),
    540
    0
                (TimestampNanosecond(_, _), _) => false,
    541
    0
                (DurationSecond(v1), DurationSecond(v2)) => v1.eq(v2),
    542
    0
                (DurationSecond(_), _) => false,
    543
    0
                (DurationMillisecond(v1), DurationMillisecond(v2)) => v1.eq(v2),
    544
    0
                (DurationMillisecond(_), _) => false,
    545
    0
                (DurationMicrosecond(v1), DurationMicrosecond(v2)) => v1.eq(v2),
    546
    0
                (DurationMicrosecond(_), _) => false,
    547
    0
                (DurationNanosecond(v1), DurationNanosecond(v2)) => v1.eq(v2),
    548
    0
                (DurationNanosecond(_), _) => false,
    549
    0
                (IntervalYearMonth(v1), IntervalYearMonth(v2)) => v1.eq(v2),
    550
    0
                (IntervalYearMonth(_), _) => false,
    551
    0
                (IntervalDayTime(v1), IntervalDayTime(v2)) => v1.eq(v2),
    552
    0
                (IntervalDayTime(_), _) => false,
    553
    0
                (IntervalMonthDayNano(v1), IntervalMonthDayNano(v2)) => v1.eq(v2),
    554
    0
                (IntervalMonthDayNano(_), _) => false,
    555
    0
                (Union(val1, fields1, mode1), Union(val2, fields2, mode2)) => {
    556
    0
                    val1.eq(val2) && fields1.eq(fields2) && mode1.eq(mode2)
    557
                }
    558
    0
                (Union(_, _, _), _) => false,
    559
    1
                (Dictionary(k1, v1), Dictionary(k2, v2)) => k1.eq(k2) && v1.eq(v2),
    560
    0
                (Dictionary(_, _), _) => false,
    561
    0
                (Null, Null) => true,
    562
    0
                (Null, _) => false,
    563
            }
    564
    2
        }
    565
    }
    566
    567
    // manual implementation of `PartialOrd`
    568
    impl PartialOrd for ScalarValue {
    569
    0
        fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
    570
            use ScalarValue::*;
    571
            // This purposely doesn't have a catch-all "(_, _)" so that
    572
            // any newly added enum variant will require editing this list
    573
            // or else face a compile error
    574
    0
            match (self, other) {
    575
    0
                (Decimal32(v1, p1, s1), Decimal32(v2, p2, s2)) => {
    576
    0
                    if p1.eq(p2) && s1.eq(s2) {
    577
    0
                        v1.partial_cmp(v2)
    578
                    } else {
    579
                        // Two decimal values can be compared if they have the same precision and scale.
    580
    0
                        None
    581
                    }
    582
                }
    583
    0
                (Decimal32(_, _, _), _) => None,
    584
    0
                (Decimal64(v1, p1, s1), Decimal64(v2, p2, s2)) => {
    585
    0
                    if p1.eq(p2) && s1.eq(s2) {
    586
    0
                        v1.partial_cmp(v2)
    587
                    } else {
    588
                        // Two decimal values can be compared if they have the same precision and scale.
    589
    0
                        None
    590
                    }
    591
                }
    592
    0
                (Decimal64(_, _, _), _) => None,
    593
    0
                (Decimal128(v1, p1, s1), Decimal128(v2, p2, s2)) => {
    594
    0
                    if p1.eq(p2) && s1.eq(s2) {
    595
    0
                        v1.partial_cmp(v2)
    596
                    } else {
    597
                        // Two decimal values can be compared if they have the same precision and scale.
    598
    0
                        None
    599
                    }
    600
                }
    601
    0
                (Decimal128(_, _, _), _) => None,
    602
    0
                (Decimal256(v1, p1, s1), Decimal256(v2, p2, s2)) => {
    603
    0
                    if p1.eq(p2) && s1.eq(s2) {
    604
    0
                        v1.partial_cmp(v2)
    605
                    } else {
    606
                        // Two decimal values can be compared if they have the same precision and scale.
    607
    0
                        None
    608
                    }
    609
                }
    610
    0
                (Decimal256(_, _, _), _) => None,
    611
    0
                (Boolean(v1), Boolean(v2)) => v1.partial_cmp(v2),
    612
    0
                (Boolean(_), _) => None,
    613
    0
                (Float32(v1), Float32(v2)) => match (v1, v2) {
    614
    0
                    (Some(f1), Some(f2)) => Some(f1.total_cmp(f2)),
    615
    0
                    _ => v1.partial_cmp(v2),
    616
                },
    617
    0
                (Float16(v1), Float16(v2)) => match (v1, v2) {
    618
    0
                    (Some(f1), Some(f2)) => Some(f1.total_cmp(f2)),
    619
    0
                    _ => v1.partial_cmp(v2),
    620
                },
    621
    0
                (Float32(_), _) => None,
    622
    0
                (Float16(_), _) => None,
    623
    0
                (Float64(v1), Float64(v2)) => match (v1, v2) {
    624
    0
                    (Some(f1), Some(f2)) => Some(f1.total_cmp(f2)),
    625
    0
                    _ => v1.partial_cmp(v2),
    626
                },
    627
    0
                (Float64(_), _) => None,
    628
    0
                (Int8(v1), Int8(v2)) => v1.partial_cmp(v2),
    629
    0
                (Int8(_), _) => None,
    630
    0
                (Int16(v1), Int16(v2)) => v1.partial_cmp(v2),
    631
    0
                (Int16(_), _) => None,
    632
    0
                (Int32(v1), Int32(v2)) => v1.partial_cmp(v2),
    633
    0
                (Int32(_), _) => None,
    634
    0
                (Int64(v1), Int64(v2)) => v1.partial_cmp(v2),
    635
    0
                (Int64(_), _) => None,
    636
    0
                (UInt8(v1), UInt8(v2)) => v1.partial_cmp(v2),
    637
    0
                (UInt8(_), _) => None,
    638
    0
                (UInt16(v1), UInt16(v2)) => v1.partial_cmp(v2),
    639
    0
                (UInt16(_), _) => None,
    640
    0
                (UInt32(v1), UInt32(v2)) => v1.partial_cmp(v2),
    641
    0
                (UInt32(_), _) => None,
    642
    0
                (UInt64(v1), UInt64(v2)) => v1.partial_cmp(v2),
    643
    0
                (UInt64(_), _) => None,
    644
    0
                (Utf8(v1), Utf8(v2)) => v1.partial_cmp(v2),
    645
    0
                (Utf8(_), _) => None,
    646
    0
                (LargeUtf8(v1), LargeUtf8(v2)) => v1.partial_cmp(v2),
    647
    0
                (LargeUtf8(_), _) => None,
    648
    0
                (Utf8View(v1), Utf8View(v2)) => v1.partial_cmp(v2),
    649
    0
                (Utf8View(_), _) => None,
    650
    0
                (Binary(v1), Binary(v2)) => v1.partial_cmp(v2),
    651
    0
                (Binary(_), _) => None,
    652
    0
                (BinaryView(v1), BinaryView(v2)) => v1.partial_cmp(v2),
    653
    0
                (BinaryView(_), _) => None,
    654
    0
                (FixedSizeBinary(_, v1), FixedSizeBinary(_, v2)) => v1.partial_cmp(v2),
    655
    0
                (FixedSizeBinary(_, _), _) => None,
    656
    0
                (LargeBinary(v1), LargeBinary(v2)) => v1.partial_cmp(v2),
    657
    0
                (LargeBinary(_), _) => None,
    658
                // ScalarValue::List / ScalarValue::FixedSizeList / ScalarValue::LargeList are ensure to have length 1
    659
    0
                (List(arr1), List(arr2)) => partial_cmp_list(arr1.as_ref(), arr2.as_ref()),
    660
    0
                (FixedSizeList(arr1), FixedSizeList(arr2)) => {
    661
    0
                    partial_cmp_list(arr1.as_ref(), arr2.as_ref())
    662
                }
    663
    0
                (LargeList(arr1), LargeList(arr2)) => {
    664
    0
                    partial_cmp_list(arr1.as_ref(), arr2.as_ref())
    665
                }
    666
    0
                (List(_), _) | (LargeList(_), _) | (FixedSizeList(_), _) => None,
    667
    0
                (Struct(struct_arr1), Struct(struct_arr2)) => {
    668
    0
                    partial_cmp_struct(struct_arr1.as_ref(), struct_arr2.as_ref())
    669
                }
    670
    0
                (Struct(_), _) => None,
    671
    0
                (Map(map_arr1), Map(map_arr2)) => partial_cmp_map(map_arr1, map_arr2),
    672
    0
                (Map(_), _) => None,
    673
    0
                (Date32(v1), Date32(v2)) => v1.partial_cmp(v2),
    674
    0
                (Date32(_), _) => None,
    675
    0
                (Date64(v1), Date64(v2)) => v1.partial_cmp(v2),
    676
    0
                (Date64(_), _) => None,
    677
    0
                (Time32Second(v1), Time32Second(v2)) => v1.partial_cmp(v2),
    678
    0
                (Time32Second(_), _) => None,
    679
    0
                (Time32Millisecond(v1), Time32Millisecond(v2)) => v1.partial_cmp(v2),
    680
    0
                (Time32Millisecond(_), _) => None,
    681
    0
                (Time64Microsecond(v1), Time64Microsecond(v2)) => v1.partial_cmp(v2),
    682
    0
                (Time64Microsecond(_), _) => None,
    683
    0
                (Time64Nanosecond(v1), Time64Nanosecond(v2)) => v1.partial_cmp(v2),
    684
    0
                (Time64Nanosecond(_), _) => None,
    685
    0
                (TimestampSecond(v1, _), TimestampSecond(v2, _)) => v1.partial_cmp(v2),
    686
    0
                (TimestampSecond(_, _), _) => None,
    687
    0
                (TimestampMillisecond(v1, _), TimestampMillisecond(v2, _)) => {
    688
    0
                    v1.partial_cmp(v2)
    689
                }
    690
    0
                (TimestampMillisecond(_, _), _) => None,
    691
    0
                (TimestampMicrosecond(v1, _), TimestampMicrosecond(v2, _)) => {
    692
    0
                    v1.partial_cmp(v2)
    693
                }
    694
    0
                (TimestampMicrosecond(_, _), _) => None,
    695
    0
                (TimestampNanosecond(v1, _), TimestampNanosecond(v2, _)) => {
    696
    0
                    v1.partial_cmp(v2)
    697
                }
    698
    0
                (TimestampNanosecond(_, _), _) => None,
    699
    0
                (IntervalYearMonth(v1), IntervalYearMonth(v2)) => v1.partial_cmp(v2),
    700
    0
                (IntervalYearMonth(_), _) => None,
    701
    0
                (IntervalDayTime(v1), IntervalDayTime(v2)) => v1.partial_cmp(v2),
    702
    0
                (IntervalDayTime(_), _) => None,
    703
    0
                (IntervalMonthDayNano(v1), IntervalMonthDayNano(v2)) => v1.partial_cmp(v2),
    704
    0
                (IntervalMonthDayNano(_), _) => None,
    705
    0
                (DurationSecond(v1), DurationSecond(v2)) => v1.partial_cmp(v2),
    706
    0
                (DurationSecond(_), _) => None,
    707
    0
                (DurationMillisecond(v1), DurationMillisecond(v2)) => v1.partial_cmp(v2),
    708
    0
                (DurationMillisecond(_), _) => None,
    709
    0
                (DurationMicrosecond(v1), DurationMicrosecond(v2)) => v1.partial_cmp(v2),
    710
    0
                (DurationMicrosecond(_), _) => None,
    711
    0
                (DurationNanosecond(v1), DurationNanosecond(v2)) => v1.partial_cmp(v2),
    712
    0
                (DurationNanosecond(_), _) => None,
    713
    0
                (Union(v1, t1, m1), Union(v2, t2, m2)) => {
    714
    0
                    if t1.eq(t2) && m1.eq(m2) {
    715
    0
                        v1.partial_cmp(v2)
    716
                    } else {
    717
    0
                        None
    718
                    }
    719
                }
    720
    0
                (Union(_, _, _), _) => None,
    721
    0
                (Dictionary(k1, v1), Dictionary(k2, v2)) => {
    722
                    // Don't compare if the key types don't match (it is effectively a different datatype)
    723
    0
                    if k1 == k2 {
    724
    0
                        v1.partial_cmp(v2)
    725
                    } else {
    726
    0
                        None
    727
                    }
    728
                }
    729
    0
                (Dictionary(_, _), _) => None,
    730
    0
                (Null, Null) => Some(Ordering::Equal),
    731
    0
                (Null, _) => None,
    732
            }
    733
    0
        }
    734
    }
    735
    736
    /// List/LargeList/FixedSizeList scalars always have a single element
    737
    /// array. This function returns that array
    738
    0
    fn first_array_for_list(arr: &dyn Array) -> ArrayRef {
    739
    0
        assert_eq!(arr.len(), 1);
    740
    0
        if let Some(arr) = arr.as_list_opt::<i32>() {
    741
    0
            arr.value(0)
    742
    0
        } else if let Some(arr) = arr.as_list_opt::<i64>() {
    743
    0
            arr.value(0)
    744
    0
        } else if let Some(arr) = arr.as_fixed_size_list_opt() {
    745
    0
            arr.value(0)
    746
        } else {
    747
    0
            unreachable!("Since only List / LargeList / FixedSizeList are supported, this should never happen")
    748
        }
    749
    0
    }
    750
    751
    /// Compares two List/LargeList/FixedSizeList scalars
    752
    0
    fn partial_cmp_list(arr1: &dyn Array, arr2: &dyn Array) -> Option<Ordering> {
    753
    0
        if arr1.data_type() != arr2.data_type() {
    754
    0
            return None;
    755
    0
        }
    756
    0
        let arr1 = first_array_for_list(arr1);
    757
    0
        let arr2 = first_array_for_list(arr2);
    758
    759
    0
        let min_length = arr1.len().min(arr2.len());
    760
    0
        let arr1_trimmed = arr1.slice(0, min_length);
    761
    0
        let arr2_trimmed = arr2.slice(0, min_length);
    762
    763
    0
        let lt_res = arrow::compute::kernels::cmp::lt(&arr1_trimmed, &arr2_trimmed).ok()?;
    764
    0
        let eq_res = arrow::compute::kernels::cmp::eq(&arr1_trimmed, &arr2_trimmed).ok()?;
    765
    766
    0
        for j in 0..lt_res.len() {
    767
            // In Postgres, NULL values in lists are always considered to be greater than non-NULL values:
    768
            //
    769
            // $ SELECT ARRAY[NULL]::integer[] > ARRAY[1]
    770
            // true
    771
            //
    772
            // These next two if statements are introduced for replicating Postgres behavior, as
    773
            // arrow::compute does not account for this.
    774
    0
            if arr1_trimmed.is_null(j) && !arr2_trimmed.is_null(j) {
    775
    0
                return Some(Ordering::Greater);
    776
    0
            }
    777
    0
            if !arr1_trimmed.is_null(j) && arr2_trimmed.is_null(j) {
    778
    0
                return Some(Ordering::Less);
    779
    0
            }
    780
    781
    0
            if lt_res.is_valid(j) && lt_res.value(j) {
    782
    0
                return Some(Ordering::Less);
    783
    0
            }
    784
    0
            if eq_res.is_valid(j) && !eq_res.value(j) {
    785
    0
                return Some(Ordering::Greater);
    786
    0
            }
    787
        }
    788
    789
    0
        Some(arr1.len().cmp(&arr2.len()))
    790
    0
    }
    791
    792
    0
    fn flatten<'a>(array: &'a StructArray, columns: &mut Vec<&'a ArrayRef>) {
    793
    0
        for i in 0..array.num_columns() {
    794
    0
            let column = array.column(i);
    795
    0
            if let Some(nested_struct) = column.as_any().downcast_ref::<StructArray>() {
    796
    0
                // If it's a nested struct, recursively expand
    797
    0
                flatten(nested_struct, columns);
    798
    0
            } else {
    799
    0
                // If it's a primitive type, add directly
    800
    0
                columns.push(column);
    801
    0
            }
    802
        }
    803
    0
    }
    804
    805
    0
    pub fn partial_cmp_struct(s1: &StructArray, s2: &StructArray) -> Option<Ordering> {
    806
    0
        if s1.len() != s2.len() {
    807
    0
            return None;
    808
    0
        }
    809
    810
    0
        if s1.data_type() != s2.data_type() {
    811
    0
            return None;
    812
    0
        }
    813
    814
    0
        let mut expanded_columns1 = Vec::with_capacity(s1.num_columns());
    815
    0
        let mut expanded_columns2 = Vec::with_capacity(s2.num_columns());
    816
    817
    0
        flatten(s1, &mut expanded_columns1);
    818
    0
        flatten(s2, &mut expanded_columns2);
    819
    820
    0
        for col_index in 0..expanded_columns1.len() {
    821
    0
            let arr1 = expanded_columns1[col_index];
    822
    0
            let arr2 = expanded_columns2[col_index];
    823
    824
    0
            let lt_res = arrow::compute::kernels::cmp::lt(arr1, arr2).ok()?;
    825
    0
            let eq_res = arrow::compute::kernels::cmp::eq(arr1, arr2).ok()?;
    826
    827
    0
            for j in 0..lt_res.len() {
    828
    0
                if lt_res.is_valid(j) && lt_res.value(j) {
    829
    0
                    return Some(Ordering::Less);
    830
    0
                }
    831
    0
                if eq_res.is_valid(j) && !eq_res.value(j) {
    832
    0
                    return Some(Ordering::Greater);
    833
    0
                }
    834
            }
    835
        }
    836
    0
        Some(Ordering::Equal)
    837
    0
    }
    838
    839
    0
    fn partial_cmp_map(m1: &Arc<MapArray>, m2: &Arc<MapArray>) -> Option<Ordering> {
    840
    0
        if m1.len() != m2.len() {
    841
    0
            return None;
    842
    0
        }
    843
    844
    0
        if m1.data_type() != m2.data_type() {
    845
    0
            return None;
    846
    0
        }
    847
    848
    0
        for col_index in 0..m1.len() {
    849
    0
            let arr1 = m1.entries().column(col_index);
    850
    0
            let arr2 = m2.entries().column(col_index);
    851
    852
    0
            let lt_res = arrow::compute::kernels::cmp::lt(arr1, arr2).ok()?;
    853
    0
            let eq_res = arrow::compute::kernels::cmp::eq(arr1, arr2).ok()?;
    854
    855
    0
            for j in 0..lt_res.len() {
    856
    0
                if lt_res.is_valid(j) && lt_res.value(j) {
    857
    0
                    return Some(Ordering::Less);
    858
    0
                }
    859
    0
                if eq_res.is_valid(j) && !eq_res.value(j) {
    860
    0
                    return Some(Ordering::Greater);
    861
    0
                }
    862
            }
    863
        }
    864
    0
        Some(Ordering::Equal)
    865
    0
    }
    866
    867
    impl Eq for ScalarValue {}
    868
    869
    //Float wrapper over f32/f64. Just because we cannot build std::hash::Hash for floats directly we have to do it through type wrapper
    870
    struct Fl<T>(T);
    871
    872
    macro_rules! hash_float_value {
    873
        ($(($t:ty, $i:ty)),+) => {
    874
            $(impl std::hash::Hash for Fl<$t> {
    875
                #[inline]
    876
    0
                fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
    877
    0
                    state.write(&<$i>::from_ne_bytes(self.0.to_ne_bytes()).to_ne_bytes())
    878
    0
                }
    879
            })+
    880
        };
    881
    }
    882
    883
    hash_float_value!((f64, u64), (f32, u32));
    884
    885
    // manual implementation of `Hash`
    886
    //
    887
    // # Panics
    888
    //
    889
    // Panics if there is an error when creating hash values for rows
    890
    impl Hash for ScalarValue {
    891
    6
        fn hash<H: Hasher>(&self, state: &mut H) {
    892
            use ScalarValue::*;
    893
    6
            match self {
    894
    0
                Decimal32(v, p, s) => {
    895
    0
                    v.hash(state);
    896
    0
                    p.hash(state);
    897
    0
                    s.hash(state)
    898
                }
    899
    0
                Decimal64(v, p, s) => {
    900
    0
                    v.hash(state);
    901
    0
                    p.hash(state);
    902
    0
                    s.hash(state)
    903
                }
    904
    0
                Decimal128(v, p, s) => {
    905
    0
                    v.hash(state);
    906
    0
                    p.hash(state);
    907
    0
                    s.hash(state)
    908
                }
    909
    0
                Decimal256(v, p, s) => {
    910
    0
                    v.hash(state);
    911
    0
                    p.hash(state);
    912
    0
                    s.hash(state)
    913
                }
    914
    0
                Boolean(v) => v.hash(state),
    915
    0
                Float16(v) => v.map(Fl).hash(state),
    916
    0
                Float32(v) => v.map(Fl).hash(state),
    917
    0
                Float64(v) => v.map(Fl).hash(state),
    918
    0
                Int8(v) => v.hash(state),
    919
    0
                Int16(v) => v.hash(state),
    920
    0
                Int32(v) => v.hash(state),
    921
    0
                Int64(v) => v.hash(state),
    922
    0
                UInt8(v) => v.hash(state),
    923
    0
                UInt16(v) => v.hash(state),
    924
    0
                UInt32(v) => v.hash(state),
    925
    0
                UInt64(v) => v.hash(state),
    926
    3
                Utf8(v) | LargeUtf8(v0) | Utf8View(v0) => v.hash(state),
    927
    0
                Binary(v) | FixedSizeBinary(_, v) | LargeBinary(v) | BinaryView(v) => {
    928
    0
                    v.hash(state)
    929
                }
    930
    0
                List(arr) => {
    931
    0
                    hash_nested_array(arr.to_owned() as ArrayRef, state);
    932
    0
                }
    933
    0
                LargeList(arr) => {
    934
    0
                    hash_nested_array(arr.to_owned() as ArrayRef, state);
    935
    0
                }
    936
    0
                FixedSizeList(arr) => {
    937
    0
                    hash_nested_array(arr.to_owned() as ArrayRef, state);
    938
    0
                }
    939
    0
                Struct(arr) => {
    940
    0
                    hash_nested_array(arr.to_owned() as ArrayRef, state);
    941
    0
                }
    942
    0
                Map(arr) => {
    943
    0
                    hash_nested_array(arr.to_owned() as ArrayRef, state);
    944
    0
                }
    945
    0
                Date32(v) => v.hash(state),
    946
    0
                Date64(v) => v.hash(state),
    947
    0
                Time32Second(v) => v.hash(state),
    948
    0
                Time32Millisecond(v) => v.hash(state),
    949
    0
                Time64Microsecond(v) => v.hash(state),
    950
    0
                Time64Nanosecond(v) => v.hash(state),
    951
    0
                TimestampSecond(v, _) => v.hash(state),
    952
    0
                TimestampMillisecond(v, _) => v.hash(state),
    953
    0
                TimestampMicrosecond(v, _) => v.hash(state),
    954
    0
                TimestampNanosecond(v, _) => v.hash(state),
    955
    0
                DurationSecond(v) => v.hash(state),
    956
    0
                DurationMillisecond(v) => v.hash(state),
    957
    0
                DurationMicrosecond(v) => v.hash(state),
    958
    0
                DurationNanosecond(v) => v.hash(state),
    959
    0
                IntervalYearMonth(v) => v.hash(state),
    960
    0
                IntervalDayTime(v) => v.hash(state),
    961
    0
                IntervalMonthDayNano(v) => v.hash(state),
    962
    0
                Union(v, t, m) => {
    963
    0
                    v.hash(state);
    964
    0
                    t.hash(state);
    965
    0
                    m.hash(state);
    966
    0
                }
    967
    3
                Dictionary(k, v) => {
    968
    3
                    k.hash(state);
    969
    3
                    v.hash(state);
    970
    3
                }
    971
                // stable hash for Null value
    972
    0
                Null => 1.hash(state),
    973
            }
    974
    6
        }
    975
    }
    976
    977
    0
    fn hash_nested_array<H: Hasher>(arr: ArrayRef, state: &mut H) {
    978
    0
        let len = arr.len();
    979
    0
        let hashes_buffer = &mut vec![0; len];
    980
    0
        let random_state = ahash::RandomState::with_seeds(0, 0, 0, 0);
    981
    0
        let hashes = create_hashes(&[arr], &random_state, hashes_buffer)
    982
    0
            .expect("hash_nested_array: failed to create row hashes");
    983
        // Hash back to std::hash::Hasher
    984
    0
        hashes.hash(state);
    985
    0
    }
    986
    987
    /// Return a reference to the values array and the index into it for a
    988
    /// dictionary array
    989
    ///
    990
    /// # Errors
    991
    ///
    992
    /// Errors if the array cannot be downcasted to DictionaryArray
    993
    #[inline]
    994
    0
    pub fn get_dict_value<K: ArrowDictionaryKeyType>(
    995
    0
        array: &dyn Array,
    996
    0
        index: usize,
    997
    0
    ) -> Result<(&ArrayRef, Option<usize>)> {
    998
    0
        let dict_array = as_dictionary_array::<K>(array)?;
    999
    0
        Ok((dict_array.values(), dict_array.key(index)))
    1000
    0
    }
    1001
    1002
    /// Create a dictionary array representing `value` repeated `size`
    1003
    /// times
    1004
    0
    fn dict_from_scalar<K: ArrowDictionaryKeyType>(
    1005
    0
        value: &ScalarValue,
    1006
    0
        size: usize,
    1007
    0
    ) -> Result<ArrayRef> {
    1008
        // values array is one element long (the value)
    1009
    0
        let values_array = value.to_array_of_size(1)?;
    1010
    1011
        // Create a key array with `size` elements, each of 0
    1012
        // Use cache to avoid repeated allocations for the same size
    1013
    0
        let key_array: PrimitiveArray<K> =
    1014
    0
            get_or_create_cached_key_array::<K>(size, value.is_null());
    1015
    1016
        // create a new DictionaryArray
    1017
        //
    1018
        // Note: this path could be made faster by using the ArrayData
    1019
        // APIs and skipping validation, if it every comes up in
    1020
        // performance traces.
    1021
    0
        Ok(Arc::new(
    1022
    0
            DictionaryArray::<K>::try_new(key_array, values_array)?, // should always be valid by construction above
    1023
        ))
    1024
    0
    }
    1025
    1026
    /// Create a `DictionaryArray` from the provided values array.
    1027
    ///
    1028
    /// Each element gets a unique key (`0..N-1`), without deduplication.
    1029
    /// Useful for wrapping arrays in dictionary form.
    1030
    ///
    1031
    /// # Input
    1032
    /// ["alice", "bob", "alice", null, "carol"]
    1033
    ///
    1034
    /// # Output
    1035
    /// `DictionaryArray<Int32>`
    1036
    /// {
    1037
    ///   keys:   [0, 1, 2, 3, 4],
    1038
    ///   values: ["alice", "bob", "alice", null, "carol"]
    1039
    /// }
    1040
    0
    pub fn dict_from_values<K: ArrowDictionaryKeyType>(
    1041
    0
        values_array: ArrayRef,
    1042
    0
    ) -> Result<ArrayRef> {
    1043
        // Create a key array with `size` elements of 0..array_len for all
    1044
        // non-null value elements
    1045
    0
        let key_array: PrimitiveArray<K> = (0..values_array.len())
    1046
    0
            .map(|index| {
    1047
    0
                if values_array.is_valid(index) {
    1048
    0
                    let native_index = K::Native::from_usize(index).ok_or_else(|| {
    1049
    0
                        _internal_datafusion_err!(
    1050
    0
                            "Can not create index of type {} from value {index}",
    1051
    0
                            K::DATA_TYPE
    1052
                        )
    1053
    0
                    })?;
    1054
    0
                    Ok(Some(native_index))
    1055
                } else {
    1056
    0
                    Ok(None)
    1057
                }
    1058
    0
            })
    1059
    0
            .collect::<Result<Vec<_>>>()?
    1060
    0
            .into_iter()
    1061
    0
            .collect();
    1062
    1063
        // create a new DictionaryArray
    1064
        //
    1065
        // Note: this path could be made faster by using the ArrayData
    1066
        // APIs and skipping validation, if it every comes up in
    1067
        // performance traces.
    1068
    0
        let dict_array = DictionaryArray::<K>::try_new(key_array, values_array)?;
    1069
    0
        Ok(Arc::new(dict_array))
    1070
    0
    }
    1071
    1072
    macro_rules! typed_cast_tz {
    1073
        ($array:expr, $index:expr, $array_cast:ident, $SCALAR:ident, $TZ:expr) => {{
    1074
            let array = $array_cast($array)?;
    1075
            Ok::<ScalarValue, DataFusionError>(ScalarValue::$SCALAR(
    1076
                match array.is_null($index) {
    1077
                    true => None,
    1078
                    false => Some(array.value($index).into()),
    1079
                },
    1080
                $TZ.clone(),
    1081
            ))
    1082
        }};
    1083
    }
    1084
    1085
    macro_rules! typed_cast {
    1086
        ($array:expr, $index:expr, $array_cast:ident, $SCALAR:ident) => {{
    1087
            let array = $array_cast($array)?;
    1088
            Ok::<ScalarValue, DataFusionError>(ScalarValue::$SCALAR(
    1089
                match array.is_null($index) {
    1090
                    true => None,
    1091
                    false => Some(array.value($index).into()),
    1092
                },
    1093
            ))
    1094
        }};
    1095
    }
    1096
    1097
    macro_rules! build_array_from_option {
    1098
        ($DATA_TYPE:ident, $ARRAY_TYPE:ident, $EXPR:expr, $SIZE:expr) => {{
    1099
            match $EXPR {
    1100
                Some(value) => Arc::new($ARRAY_TYPE::from_value(*value, $SIZE)),
    1101
                None => new_null_array(&DataType::$DATA_TYPE, $SIZE),
    1102
            }
    1103
        }};
    1104
        ($DATA_TYPE:ident, $ENUM:expr, $ARRAY_TYPE:ident, $EXPR:expr, $SIZE:expr) => {{
    1105
            match $EXPR {
    1106
                Some(value) => Arc::new($ARRAY_TYPE::from_value(*value, $SIZE)),
    1107
                None => new_null_array(&DataType::$DATA_TYPE($ENUM), $SIZE),
    1108
            }
    1109
        }};
    1110
    }
    1111
    1112
    macro_rules! build_timestamp_array_from_option {
    1113
        ($TIME_UNIT:expr, $TZ:expr, $ARRAY_TYPE:ident, $EXPR:expr, $SIZE:expr) => {
    1114
            match $EXPR {
    1115
                Some(value) => {
    1116
                    Arc::new($ARRAY_TYPE::from_value(*value, $SIZE).with_timezone_opt($TZ))
    1117
                }
    1118
                None => new_null_array(&DataType::Timestamp($TIME_UNIT, $TZ), $SIZE),
    1119
            }
    1120
        };
    1121
    }
    1122
    1123
    macro_rules! eq_array_primitive {
    1124
        ($array:expr, $index:expr, $array_cast:ident, $VALUE:expr) => {{
    1125
            let array = $array_cast($array)?;
    1126
            let is_valid = array.is_valid($index);
    1127
            Ok::<bool, DataFusionError>(match $VALUE {
    1128
                Some(val) => is_valid && &array.value($index) == val,
    1129
                None => !is_valid,
    1130
            })
    1131
        }};
    1132
    }
    1133
    1134
    impl ScalarValue {
    1135
        /// Create a [`Result<ScalarValue>`] with the provided value and datatype
    1136
        ///
    1137
        /// # Panics
    1138
        ///
    1139
        /// Panics if d is not compatible with T
    1140
    0
        pub fn new_primitive<T: ArrowPrimitiveType>(
    1141
    0
            a: Option<T::Native>,
    1142
    0
            d: &DataType,
    1143
    0
        ) -> Result<Self> {
    1144
    0
            match a {
    1145
    0
                None => d.try_into(),
    1146
    0
                Some(v) => {
    1147
    0
                    let array = PrimitiveArray::<T>::new(vec![v].into(), None)
    1148
    0
                        .with_data_type(d.clone());
    1149
    0
                    Self::try_from_array(&array, 0)
    1150
                }
    1151
            }
    1152
    0
        }
    1153
    1154
        /// Create a decimal Scalar from value/precision and scale.
    1155
    0
        pub fn try_new_decimal128(value: i128, precision: u8, scale: i8) -> Result<Self> {
    1156
            // make sure the precision and scale is valid
    1157
    0
            if precision <= DECIMAL128_MAX_PRECISION && scale.unsigned_abs() <= precision {
    1158
    0
                return Ok(ScalarValue::Decimal128(Some(value), precision, scale));
    1159
    0
            }
    1160
    0
            _internal_err!(
    1161
    0
                "Can not new a decimal type ScalarValue for precision {precision} and scale {scale}"
    1162
            )
    1163
    0
        }
    1164
    1165
        /// Create a Null instance of ScalarValue for this datatype
    1166
        ///
    1167
        /// Example
    1168
        /// ```
    1169
        /// use arrow::datatypes::DataType;
    1170
        /// use datafusion_common::ScalarValue;
    1171
        ///
    1172
        /// let scalar = ScalarValue::try_new_null(&DataType::Int32).unwrap();
    1173
        /// assert_eq!(scalar.is_null(), true);
    1174
        /// assert_eq!(scalar.data_type(), DataType::Int32);
    1175
        /// ```
    1176
    31
        pub fn try_new_null(data_type: &DataType) -> Result<Self> {
    1177
    0
            Ok(match data_type {
    1178
    2
                DataType::Boolean => ScalarValue::Boolean(None),
    1179
    0
                DataType::Float16 => ScalarValue::Float16(None),
    1180
    2
                DataType::Float64 => ScalarValue::Float64(None),
    1181
    0
                DataType::Float32 => ScalarValue::Float32(None),
    1182
    2
                DataType::Int8 => ScalarValue::Int8(None),
    1183
    2
                DataType::Int16 => ScalarValue::Int16(None),
    1184
    2
                DataType::Int32 => ScalarValue::Int32(None),
    1185
    7
                DataType::Int64 => ScalarValue::Int64(None),
    1186
    2
                DataType::UInt8 => ScalarValue::UInt8(None),
    1187
    2
                DataType::UInt16 => ScalarValue::UInt16(None),
    1188
    2
                DataType::UInt32 => ScalarValue::UInt32(None),
    1189
    2
                DataType::UInt64 => ScalarValue::UInt64(None),
    1190
    0
                DataType::Decimal32(precision, scale) => {
    1191
    0
                    ScalarValue::Decimal32(None, *precision, *scale)
    1192
                }
    1193
    0
                DataType::Decimal64(precision, scale) => {
    1194
    0
                    ScalarValue::Decimal64(None, *precision, *scale)
    1195
                }
    1196
    2
                DataType::Decimal128(precision, scale) => {
    1197
    2
                    ScalarValue::Decimal128(None, *precision, *scale)
    1198
                }
    1199
    0
                DataType::Decimal256(precision, scale) => {
    1200
    0
                    ScalarValue::Decimal256(None, *precision, *scale)
    1201
                }
    1202
    0
                DataType::Utf8 => ScalarValue::Utf8(None),
    1203
    0
                DataType::LargeUtf8 => ScalarValue::LargeUtf8(None),
    1204
    0
                DataType::Utf8View => ScalarValue::Utf8View(None),
    1205
    0
                DataType::Binary => ScalarValue::Binary(None),
    1206
    0
                DataType::BinaryView => ScalarValue::BinaryView(None),
    1207
    0
                DataType::FixedSizeBinary(len) => ScalarValue::FixedSizeBinary(*len, None),
    1208
    0
                DataType::LargeBinary => ScalarValue::LargeBinary(None),
    1209
    2
                DataType::Date32 => ScalarValue::Date32(None),
    1210
    2
                DataType::Date64 => ScalarValue::Date64(None),
    1211
    0
                DataType::Time32(TimeUnit::Second) => ScalarValue::Time32Second(None),
    1212
                DataType::Time32(TimeUnit::Millisecond) => {
    1213
    0
                    ScalarValue::Time32Millisecond(None)
    1214
                }
    1215
                DataType::Time64(TimeUnit::Microsecond) => {
    1216
    0
                    ScalarValue::Time64Microsecond(None)
    1217
                }
    1218
    0
                DataType::Time64(TimeUnit::Nanosecond) => ScalarValue::Time64Nanosecond(None),
    1219
    0
                DataType::Timestamp(TimeUnit::Second, tz_opt) => {
    1220
    0
                    ScalarValue::TimestampSecond(None, tz_opt.clone())
    1221
                }
    1222
    0
                DataType::Timestamp(TimeUnit::Millisecond, tz_opt) => {
    1223
    0
                    ScalarValue::TimestampMillisecond(None, tz_opt.clone())
    1224
                }
    1225
    0
                DataType::Timestamp(TimeUnit::Microsecond, tz_opt) => {
    1226
    0
                    ScalarValue::TimestampMicrosecond(None, tz_opt.clone())
    1227
                }
    1228
    0
                DataType::Timestamp(TimeUnit::Nanosecond, tz_opt) => {
    1229
    0
                    ScalarValue::TimestampNanosecond(None, tz_opt.clone())
    1230
                }
    1231
                DataType::Interval(IntervalUnit::YearMonth) => {
    1232
    0
                    ScalarValue::IntervalYearMonth(None)
    1233
                }
    1234
                DataType::Interval(IntervalUnit::DayTime) => {
    1235
    0
                    ScalarValue::IntervalDayTime(None)
    1236
                }
    1237
                DataType::Interval(IntervalUnit::MonthDayNano) => {
    1238
    0
                    ScalarValue::IntervalMonthDayNano(None)
    1239
                }
    1240
    0
                DataType::Duration(TimeUnit::Second) => ScalarValue::DurationSecond(None),
    1241
                DataType::Duration(TimeUnit::Millisecond) => {
    1242
    0
                    ScalarValue::DurationMillisecond(None)
    1243
                }
    1244
                DataType::Duration(TimeUnit::Microsecond) => {
    1245
    0
                    ScalarValue::DurationMicrosecond(None)
    1246
                }
    1247
                DataType::Duration(TimeUnit::Nanosecond) => {
    1248
    0
                    ScalarValue::DurationNanosecond(None)
    1249
                }
    1250
    0
                DataType::Dictionary(index_type, value_type) => ScalarValue::Dictionary(
    1251
    0
                    index_type.clone(),
    1252
    0
                    Box::new(value_type.as_ref().try_into()?),
    1253
                ),
    1254
                // `ScalaValue::List` contains single element `ListArray`.
    1255
    0
                DataType::List(field_ref) => ScalarValue::List(Arc::new(
    1256
    0
                    GenericListArray::new_null(Arc::clone(field_ref), 1),
    1257
    0
                )),
    1258
                // `ScalarValue::LargeList` contains single element `LargeListArray`.
    1259
    0
                DataType::LargeList(field_ref) => ScalarValue::LargeList(Arc::new(
    1260
    0
                    GenericListArray::new_null(Arc::clone(field_ref), 1),
    1261
    0
                )),
    1262
                // `ScalaValue::FixedSizeList` contains single element `FixedSizeList`.
    1263
    0
                DataType::FixedSizeList(field_ref, fixed_length) => {
    1264
    0
                    ScalarValue::FixedSizeList(Arc::new(FixedSizeListArray::new_null(
    1265
    0
                        Arc::clone(field_ref),
    1266
    0
                        *fixed_length,
    1267
    0
                        1,
    1268
    0
                    )))
    1269
                }
    1270
    0
                DataType::Struct(fields) => ScalarValue::Struct(
    1271
    0
                    new_null_array(&DataType::Struct(fields.to_owned()), 1)
    1272
    0
                        .as_struct()
    1273
    0
                        .to_owned()
    1274
    0
                        .into(),
    1275
    0
                ),
    1276
    0
                DataType::Map(fields, sorted) => ScalarValue::Map(
    1277
    0
                    new_null_array(&DataType::Map(fields.to_owned(), sorted.to_owned()), 1)
    1278
    0
                        .as_map()
    1279
    0
                        .to_owned()
    1280
    0
                        .into(),
    1281
    0
                ),
    1282
    0
                DataType::Union(fields, mode) => {
    1283
    0
                    ScalarValue::Union(None, fields.clone(), *mode)
    1284
                }
    1285
    0
                DataType::Null => ScalarValue::Null,
    1286
                _ => {
    1287
    0
                    return _not_impl_err!(
    1288
    0
                        "Can't create a null scalar from data_type \"{data_type}\""
    1289
                    );
    1290
                }
    1291
            })
    1292
    31
        }
    1293
    1294
        /// Returns a [`ScalarValue::Utf8`] representing `val`
    1295
    4
        pub fn new_utf8(val: impl Into<String>) -> Self {
    1296
    4
            ScalarValue::from(val.into())
    1297
    4
        }
    1298
    1299
        /// Returns a [`ScalarValue::Utf8View`] representing `val`
    1300
    0
        pub fn new_utf8view(val: impl Into<String>) -> Self {
    1301
    0
            ScalarValue::Utf8View(Some(val.into()))
    1302
    0
        }
    1303
    1304
        /// Returns a [`ScalarValue::IntervalYearMonth`] representing
    1305
        /// `years` years and `months` months
    1306
    0
        pub fn new_interval_ym(years: i32, months: i32) -> Self {
    1307
    0
            let val = IntervalYearMonthType::make_value(years, months);
    1308
    0
            ScalarValue::IntervalYearMonth(Some(val))
    1309
    0
        }
    1310
    1311
        /// Returns a [`ScalarValue::IntervalDayTime`] representing
    1312
        /// `days` days and `millis` milliseconds
    1313
    0
        pub fn new_interval_dt(days: i32, millis: i32) -> Self {
    1314
    0
            let val = IntervalDayTimeType::make_value(days, millis);
    1315
    0
            Self::IntervalDayTime(Some(val))
    1316
    0
        }
    1317
    1318
        /// Returns a [`ScalarValue::IntervalMonthDayNano`] representing
    1319
        /// `months` months and `days` days, and `nanos` nanoseconds
    1320
    0
        pub fn new_interval_mdn(months: i32, days: i32, nanos: i64) -> Self {
    1321
    0
            let val = IntervalMonthDayNanoType::make_value(months, days, nanos);
    1322
    0
            ScalarValue::IntervalMonthDayNano(Some(val))
    1323
    0
        }
    1324
    1325
        /// Returns a [`ScalarValue`] representing
    1326
        /// `value` and `tz_opt` timezone
    1327
    0
        pub fn new_timestamp<T: ArrowTimestampType>(
    1328
    0
            value: Option<i64>,
    1329
    0
            tz_opt: Option<Arc<str>>,
    1330
    0
        ) -> Self {
    1331
    0
            match T::UNIT {
    1332
    0
                TimeUnit::Second => ScalarValue::TimestampSecond(value, tz_opt),
    1333
    0
                TimeUnit::Millisecond => ScalarValue::TimestampMillisecond(value, tz_opt),
    1334
    0
                TimeUnit::Microsecond => ScalarValue::TimestampMicrosecond(value, tz_opt),
    1335
    0
                TimeUnit::Nanosecond => ScalarValue::TimestampNanosecond(value, tz_opt),
    1336
            }
    1337
    0
        }
    1338
    1339
        /// Returns a [`ScalarValue`] representing PI
    1340
    0
        pub fn new_pi(datatype: &DataType) -> Result<ScalarValue> {
    1341
    0
            match datatype {
    1342
    0
                DataType::Float32 => Ok(ScalarValue::from(std::f32::consts::PI)),
    1343
    0
                DataType::Float64 => Ok(ScalarValue::from(std::f64::consts::PI)),
    1344
    0
                _ => _internal_err!("PI is not supported for data type: {}", datatype),
    1345
            }
    1346
    0
        }
    1347
    1348
        /// Returns a [`ScalarValue`] representing PI's upper bound
    1349
    0
        pub fn new_pi_upper(datatype: &DataType) -> Result<ScalarValue> {
    1350
    0
            match datatype {
    1351
    0
                DataType::Float32 => Ok(ScalarValue::from(consts::PI_UPPER_F32)),
    1352
    0
                DataType::Float64 => Ok(ScalarValue::from(consts::PI_UPPER_F64)),
    1353
                _ => {
    1354
    0
                    _internal_err!("PI_UPPER is not supported for data type: {}", datatype)
    1355
                }
    1356
            }
    1357
    0
        }
    1358
    1359
        /// Returns a [`ScalarValue`] representing -PI's lower bound
    1360
    0
        pub fn new_negative_pi_lower(datatype: &DataType) -> Result<ScalarValue> {
    1361
    0
            match datatype {
    1362
    0
                DataType::Float32 => Ok(ScalarValue::from(consts::NEGATIVE_PI_LOWER_F32)),
    1363
    0
                DataType::Float64 => Ok(ScalarValue::from(consts::NEGATIVE_PI_LOWER_F64)),
    1364
                _ => {
    1365
    0
                    _internal_err!("-PI_LOWER is not supported for data type: {}", datatype)
    1366
                }
    1367
            }
    1368
    0
        }
    1369
    1370
        /// Returns a [`ScalarValue`] representing FRAC_PI_2's upper bound
    1371
    0
        pub fn new_frac_pi_2_upper(datatype: &DataType) -> Result<ScalarValue> {
    1372
    0
            match datatype {
    1373
    0
                DataType::Float32 => Ok(ScalarValue::from(consts::FRAC_PI_2_UPPER_F32)),
    1374
    0
                DataType::Float64 => Ok(ScalarValue::from(consts::FRAC_PI_2_UPPER_F64)),
    1375
                _ => {
    1376
    0
                    _internal_err!("PI_UPPER/2 is not supported for data type: {}", datatype)
    1377
                }
    1378
            }
    1379
    0
        }
    1380
    1381
        // Returns a [`ScalarValue`] representing FRAC_PI_2's lower bound
    1382
    0
        pub fn new_neg_frac_pi_2_lower(datatype: &DataType) -> Result<ScalarValue> {
    1383
    0
            match datatype {
    1384
                DataType::Float32 => {
    1385
    0
                    Ok(ScalarValue::from(consts::NEGATIVE_FRAC_PI_2_LOWER_F32))
    1386
                }
    1387
                DataType::Float64 => {
    1388
    0
                    Ok(ScalarValue::from(consts::NEGATIVE_FRAC_PI_2_LOWER_F64))
    1389
                }
    1390
                _ => {
    1391
    0
                    _internal_err!("-PI/2_LOWER is not supported for data type: {}", datatype)
    1392
                }
    1393
            }
    1394
    0
        }
    1395
    1396
        /// Returns a [`ScalarValue`] representing -PI
    1397
    0
        pub fn new_negative_pi(datatype: &DataType) -> Result<ScalarValue> {
    1398
    0
            match datatype {
    1399
    0
                DataType::Float32 => Ok(ScalarValue::from(-std::f32::consts::PI)),
    1400
    0
                DataType::Float64 => Ok(ScalarValue::from(-std::f64::consts::PI)),
    1401
    0
                _ => _internal_err!("-PI is not supported for data type: {}", datatype),
    1402
            }
    1403
    0
        }
    1404
    1405
        /// Returns a [`ScalarValue`] representing PI/2
    1406
    0
        pub fn new_frac_pi_2(datatype: &DataType) -> Result<ScalarValue> {
    1407
    0
            match datatype {
    1408
    0
                DataType::Float32 => Ok(ScalarValue::from(std::f32::consts::FRAC_PI_2)),
    1409
    0
                DataType::Float64 => Ok(ScalarValue::from(std::f64::consts::FRAC_PI_2)),
    1410
    0
                _ => _internal_err!("PI/2 is not supported for data type: {}", datatype),
    1411
            }
    1412
    0
        }
    1413
    1414
        /// Returns a [`ScalarValue`] representing -PI/2
    1415
    0
        pub fn new_neg_frac_pi_2(datatype: &DataType) -> Result<ScalarValue> {
    1416
    0
            match datatype {
    1417
    0
                DataType::Float32 => Ok(ScalarValue::from(-std::f32::consts::FRAC_PI_2)),
    1418
    0
                DataType::Float64 => Ok(ScalarValue::from(-std::f64::consts::FRAC_PI_2)),
    1419
    0
                _ => _internal_err!("-PI/2 is not supported for data type: {}", datatype),
    1420
            }
    1421
    0
        }
    1422
    1423
        /// Returns a [`ScalarValue`] representing infinity
    1424
    0
        pub fn new_infinity(datatype: &DataType) -> Result<ScalarValue> {
    1425
    0
            match datatype {
    1426
    0
                DataType::Float32 => Ok(ScalarValue::from(f32::INFINITY)),
    1427
    0
                DataType::Float64 => Ok(ScalarValue::from(f64::INFINITY)),
    1428
                _ => {
    1429
    0
                    _internal_err!("Infinity is not supported for data type: {}", datatype)
    1430
                }
    1431
            }
    1432
    0
        }
    1433
    1434
        /// Returns a [`ScalarValue`] representing negative infinity
    1435
    0
        pub fn new_neg_infinity(datatype: &DataType) -> Result<ScalarValue> {
    1436
    0
            match datatype {
    1437
    0
                DataType::Float32 => Ok(ScalarValue::from(f32::NEG_INFINITY)),
    1438
    0
                DataType::Float64 => Ok(ScalarValue::from(f64::NEG_INFINITY)),
    1439
                _ => {
    1440
    0
                    _internal_err!(
    1441
    0
                        "Negative Infinity is not supported for data type: {}",
    1442
                        datatype
    1443
                    )
    1444
                }
    1445
            }
    1446
    0
        }
    1447
    1448
        /// Create a zero value in the given type.
    1449
    0
        pub fn new_zero(datatype: &DataType) -> Result<ScalarValue> {
    1450
    0
            Ok(match datatype {
    1451
    0
                DataType::Boolean => ScalarValue::Boolean(Some(false)),
    1452
    0
                DataType::Int8 => ScalarValue::Int8(Some(0)),
    1453
    0
                DataType::Int16 => ScalarValue::Int16(Some(0)),
    1454
    0
                DataType::Int32 => ScalarValue::Int32(Some(0)),
    1455
    0
                DataType::Int64 => ScalarValue::Int64(Some(0)),
    1456
    0
                DataType::UInt8 => ScalarValue::UInt8(Some(0)),
    1457
    0
                DataType::UInt16 => ScalarValue::UInt16(Some(0)),
    1458
    0
                DataType::UInt32 => ScalarValue::UInt32(Some(0)),
    1459
    0
                DataType::UInt64 => ScalarValue::UInt64(Some(0)),
    1460
    0
                DataType::Float16 => ScalarValue::Float16(Some(f16::from_f32(0.0))),
    1461
    0
                DataType::Float32 => ScalarValue::Float32(Some(0.0)),
    1462
    0
                DataType::Float64 => ScalarValue::Float64(Some(0.0)),
    1463
    0
                DataType::Decimal32(precision, scale) => {
    1464
    0
                    ScalarValue::Decimal32(Some(0), *precision, *scale)
    1465
                }
    1466
    0
                DataType::Decimal64(precision, scale) => {
    1467
    0
                    ScalarValue::Decimal64(Some(0), *precision, *scale)
    1468
                }
    1469
    0
                DataType::Decimal128(precision, scale) => {
    1470
    0
                    ScalarValue::Decimal128(Some(0), *precision, *scale)
    1471
                }
    1472
    0
                DataType::Decimal256(precision, scale) => {
    1473
    0
                    ScalarValue::Decimal256(Some(i256::ZERO), *precision, *scale)
    1474
                }
    1475
    0
                DataType::Timestamp(TimeUnit::Second, tz) => {
    1476
    0
                    ScalarValue::TimestampSecond(Some(0), tz.clone())
    1477
                }
    1478
    0
                DataType::Timestamp(TimeUnit::Millisecond, tz) => {
    1479
    0
                    ScalarValue::TimestampMillisecond(Some(0), tz.clone())
    1480
                }
    1481
    0
                DataType::Timestamp(TimeUnit::Microsecond, tz) => {
    1482
    0
                    ScalarValue::TimestampMicrosecond(Some(0), tz.clone())
    1483
                }
    1484
    0
                DataType::Timestamp(TimeUnit::Nanosecond, tz) => {
    1485
    0
                    ScalarValue::TimestampNanosecond(Some(0), tz.clone())
    1486
                }
    1487
    0
                DataType::Time32(TimeUnit::Second) => ScalarValue::Time32Second(Some(0)),
    1488
                DataType::Time32(TimeUnit::Millisecond) => {
    1489
    0
                    ScalarValue::Time32Millisecond(Some(0))
    1490
                }
    1491
                DataType::Time64(TimeUnit::Microsecond) => {
    1492
    0
                    ScalarValue::Time64Microsecond(Some(0))
    1493
                }
    1494
                DataType::Time64(TimeUnit::Nanosecond) => {
    1495
    0
                    ScalarValue::Time64Nanosecond(Some(0))
    1496
                }
    1497
                DataType::Interval(IntervalUnit::YearMonth) => {
    1498
    0
                    ScalarValue::IntervalYearMonth(Some(0))
    1499
                }
    1500
                DataType::Interval(IntervalUnit::DayTime) => {
    1501
    0
                    ScalarValue::IntervalDayTime(Some(IntervalDayTime::ZERO))
    1502
                }
    1503
                DataType::Interval(IntervalUnit::MonthDayNano) => {
    1504
    0
                    ScalarValue::IntervalMonthDayNano(Some(IntervalMonthDayNano::ZERO))
    1505
                }
    1506
    0
                DataType::Duration(TimeUnit::Second) => ScalarValue::DurationSecond(Some(0)),
    1507
                DataType::Duration(TimeUnit::Millisecond) => {
    1508
    0
                    ScalarValue::DurationMillisecond(Some(0))
    1509
                }
    1510
                DataType::Duration(TimeUnit::Microsecond) => {
    1511
    0
                    ScalarValue::DurationMicrosecond(Some(0))
    1512
                }
    1513
                DataType::Duration(TimeUnit::Nanosecond) => {
    1514
    0
                    ScalarValue::DurationNanosecond(Some(0))
    1515
                }
    1516
    0
                DataType::Date32 => ScalarValue::Date32(Some(0)),
    1517
    0
                DataType::Date64 => ScalarValue::Date64(Some(0)),
    1518
                _ => {
    1519
    0
                    return _not_impl_err!(
    1520
    0
                        "Can't create a zero scalar from data_type \"{datatype}\""
    1521
                    );
    1522
                }
    1523
            })
    1524
    0
        }
    1525
    1526
        /// Returns a default value for the given `DataType`.
    1527
        ///
    1528
        /// This function is useful when you need to initialize a column with
    1529
        /// non-null values in a DataFrame or when you need a "zero" value
    1530
        /// for a specific data type.
    1531
        ///
    1532
        /// # Default Values
    1533
        ///
    1534
        /// - **Numeric types**: Returns zero (via [`new_zero`])
    1535
        /// - **String types**: Returns empty string (`""`)
    1536
        /// - **Binary types**: Returns empty byte array
    1537
        /// - **Temporal types**: Returns zero/epoch value
    1538
        /// - **List types**: Returns empty list
    1539
        /// - **Struct types**: Returns struct with all fields set to their defaults
    1540
        /// - **Dictionary types**: Returns dictionary with default value
    1541
        /// - **Map types**: Returns empty map
    1542
        /// - **Union types**: Returns first variant with default value
    1543
        ///
    1544
        /// # Errors
    1545
        ///
    1546
        /// Returns an error for data types that don't have a clear default value
    1547
        /// or are not yet supported (e.g., `RunEndEncoded`).
    1548
        ///
    1549
        /// [`new_zero`]: Self::new_zero
    1550
    0
        pub fn new_default(datatype: &DataType) -> Result<ScalarValue> {
    1551
    0
            match datatype {
    1552
                // Null type
    1553
    0
                DataType::Null => Ok(ScalarValue::Null),
    1554
    1555
                // Numeric types
    1556
                DataType::Boolean
    1557
                | DataType::Int8
    1558
                | DataType::Int16
    1559
                | DataType::Int32
    1560
                | DataType::Int64
    1561
                | DataType::UInt8
    1562
                | DataType::UInt16
    1563
                | DataType::UInt32
    1564
                | DataType::UInt64
    1565
                | DataType::Float16
    1566
                | DataType::Float32
    1567
                | DataType::Float64
    1568
                | DataType::Decimal128(_, _)
    1569
                | DataType::Decimal256(_, _)
    1570
                | DataType::Timestamp(_, _)
    1571
                | DataType::Time32(_)
    1572
                | DataType::Time64(_)
    1573
                | DataType::Interval(_)
    1574
                | DataType::Duration(_)
    1575
                | DataType::Date32
    1576
    0
                | DataType::Date64 => ScalarValue::new_zero(datatype),
    1577
    1578
                // String types
    1579
    0
                DataType::Utf8 => Ok(ScalarValue::Utf8(Some("".to_string()))),
    1580
    0
                DataType::LargeUtf8 => Ok(ScalarValue::LargeUtf8(Some("".to_string()))),
    1581
    0
                DataType::Utf8View => Ok(ScalarValue::Utf8View(Some("".to_string()))),
    1582
    1583
                // Binary types
    1584
    0
                DataType::Binary => Ok(ScalarValue::Binary(Some(vec![]))),
    1585
    0
                DataType::LargeBinary => Ok(ScalarValue::LargeBinary(Some(vec![]))),
    1586
    0
                DataType::BinaryView => Ok(ScalarValue::BinaryView(Some(vec![]))),
    1587
    1588
                // Fixed-size binary
    1589
    0
                DataType::FixedSizeBinary(size) => Ok(ScalarValue::FixedSizeBinary(
    1590
    0
                    *size,
    1591
    0
                    Some(vec![0; *size as usize]),
    1592
    0
                )),
    1593
    1594
                // List types
    1595
    0
                DataType::List(field) => {
    1596
    0
                    let list =
    1597
    0
                        ScalarValue::new_list(&[], field.data_type(), field.is_nullable());
    1598
    0
                    Ok(ScalarValue::List(list))
    1599
                }
    1600
    0
                DataType::FixedSizeList(field, _size) => {
    1601
    0
                    let empty_arr = new_empty_array(field.data_type());
    1602
    0
                    let values = Arc::new(
    1603
    0
                        SingleRowListArrayBuilder::new(empty_arr)
    1604
    0
                            .with_nullable(field.is_nullable())
    1605
    0
                            .build_fixed_size_list_array(0),
    1606
                    );
    1607
    0
                    Ok(ScalarValue::FixedSizeList(values))
    1608
                }
    1609
    0
                DataType::LargeList(field) => {
    1610
    0
                    let list = ScalarValue::new_large_list(&[], field.data_type());
    1611
    0
                    Ok(ScalarValue::LargeList(list))
    1612
                }
    1613
    1614
                // Struct types
    1615
    0
                DataType::Struct(fields) => {
    1616
    0
                    let values = fields
    1617
    0
                        .iter()
    1618
    0
                        .map(|f| ScalarValue::new_default(f.data_type()))
    1619
    0
                        .collect::<Result<Vec<_>>>()?;
    1620
    0
                    Ok(ScalarValue::Struct(Arc::new(StructArray::new(
    1621
    0
                        fields.clone(),
    1622
    0
                        values
    1623
    0
                            .into_iter()
    1624
    0
                            .map(|v| v.to_array())
    1625
    0
                            .collect::<Result<_>>()?,
    1626
    0
                        None,
    1627
                    ))))
    1628
                }
    1629
    1630
                // Dictionary types
    1631
    0
                DataType::Dictionary(key_type, value_type) => Ok(ScalarValue::Dictionary(
    1632
    0
                    key_type.clone(),
    1633
    0
                    Box::new(ScalarValue::new_default(value_type)?),
    1634
                )),
    1635
    1636
                // Map types
    1637
    0
                DataType::Map(field, _) => Ok(ScalarValue::Map(Arc::new(MapArray::from(
    1638
    0
                    ArrayData::new_empty(field.data_type()),
    1639
    0
                )))),
    1640
    1641
                // Union types - return first variant with default value
    1642
    0
                DataType::Union(fields, mode) => {
    1643
    0
                    if let Some((type_id, field)) = fields.iter().next() {
    1644
    0
                        let default_value = ScalarValue::new_default(field.data_type())?;
    1645
    0
                        Ok(ScalarValue::Union(
    1646
    0
                            Some((type_id, Box::new(default_value))),
    1647
    0
                            fields.clone(),
    1648
    0
                            *mode,
    1649
    0
                        ))
    1650
                    } else {
    1651
    0
                        _internal_err!("Union type must have at least one field")
    1652
                    }
    1653
                }
    1654
    1655
                // Unsupported types for now
    1656
                _ => {
    1657
    0
                    _not_impl_err!(
    1658
    0
                        "Default value for data_type \"{datatype}\" is not implemented yet"
    1659
                    )
    1660
                }
    1661
            }
    1662
    0
        }
    1663
    1664
        /// Create an one value in the given type.
    1665
    0
        pub fn new_one(datatype: &DataType) -> Result<ScalarValue> {
    1666
    0
            Ok(match datatype {
    1667
    0
                DataType::Int8 => ScalarValue::Int8(Some(1)),
    1668
    0
                DataType::Int16 => ScalarValue::Int16(Some(1)),
    1669
    0
                DataType::Int32 => ScalarValue::Int32(Some(1)),
    1670
    0
                DataType::Int64 => ScalarValue::Int64(Some(1)),
    1671
    0
                DataType::UInt8 => ScalarValue::UInt8(Some(1)),
    1672
    0
                DataType::UInt16 => ScalarValue::UInt16(Some(1)),
    1673
    0
                DataType::UInt32 => ScalarValue::UInt32(Some(1)),
    1674
    0
                DataType::UInt64 => ScalarValue::UInt64(Some(1)),
    1675
    0
                DataType::Float16 => ScalarValue::Float16(Some(f16::from_f32(1.0))),
    1676
    0
                DataType::Float32 => ScalarValue::Float32(Some(1.0)),
    1677
    0
                DataType::Float64 => ScalarValue::Float64(Some(1.0)),
    1678
    0
                DataType::Decimal32(precision, scale) => {
    1679
    0
                    Self::validate_decimal_or_internal_err::<Decimal32Type>(
    1680
    0
                        *precision, *scale,
    1681
    0
                    )?;
    1682
    0
                    assert_or_internal_err!(*scale >= 0, "Negative scale is not supported");
    1683
    0
                    match 10_i32.checked_pow(*scale as u32) {
    1684
    0
                        Some(value) => {
    1685
    0
                            ScalarValue::Decimal32(Some(value), *precision, *scale)
    1686
                        }
    1687
    0
                        None => return _internal_err!("Unsupported scale {scale}"),
    1688
                    }
    1689
                }
    1690
    0
                DataType::Decimal64(precision, scale) => {
    1691
    0
                    Self::validate_decimal_or_internal_err::<Decimal64Type>(
    1692
    0
                        *precision, *scale,
    1693
    0
                    )?;
    1694
    0
                    assert_or_internal_err!(*scale >= 0, "Negative scale is not supported");
    1695
    0
                    match i64::from(10).checked_pow(*scale as u32) {
    1696
    0
                        Some(value) => {
    1697
    0
                            ScalarValue::Decimal64(Some(value), *precision, *scale)
    1698
                        }
    1699
    0
                        None => return _internal_err!("Unsupported scale {scale}"),
    1700
                    }
    1701
                }
    1702
    0
                DataType::Decimal128(precision, scale) => {
    1703
    0
                    Self::validate_decimal_or_internal_err::<Decimal128Type>(
    1704
    0
                        *precision, *scale,
    1705
    0
                    )?;
    1706
    0
                    assert_or_internal_err!(*scale >= 0, "Negative scale is not supported");
    1707
    0
                    match i128::from(10).checked_pow(*scale as u32) {
    1708
    0
                        Some(value) => {
    1709
    0
                            ScalarValue::Decimal128(Some(value), *precision, *scale)
    1710
                        }
    1711
    0
                        None => return _internal_err!("Unsupported scale {scale}"),
    1712
                    }
    1713
                }
    1714
    0
                DataType::Decimal256(precision, scale) => {
    1715
    0
                    Self::validate_decimal_or_internal_err::<Decimal256Type>(
    1716
    0
                        *precision, *scale,
    1717
    0
                    )?;
    1718
    0
                    assert_or_internal_err!(*scale >= 0, "Negative scale is not supported");
    1719
    0
                    match i256::from(10).checked_pow(*scale as u32) {
    1720
    0
                        Some(value) => {
    1721
    0
                            ScalarValue::Decimal256(Some(value), *precision, *scale)
    1722
                        }
    1723
    0
                        None => return _internal_err!("Unsupported scale {scale}"),
    1724
                    }
    1725
                }
    1726
                _ => {
    1727
    0
                    return _not_impl_err!(
    1728
    0
                        "Can't create an one scalar from data_type \"{datatype}\""
    1729
                    );
    1730
                }
    1731
            })
    1732
    0
        }
    1733
    1734
        /// Create a negative one value in the given type.
    1735
    0
        pub fn new_negative_one(datatype: &DataType) -> Result<ScalarValue> {
    1736
    0
            Ok(match datatype {
    1737
    0
                DataType::Int8 | DataType::UInt8 => ScalarValue::Int8(Some(-1)),
    1738
    0
                DataType::Int16 | DataType::UInt16 => ScalarValue::Int16(Some(-1)),
    1739
    0
                DataType::Int32 | DataType::UInt32 => ScalarValue::Int32(Some(-1)),
    1740
    0
                DataType::Int64 | DataType::UInt64 => ScalarValue::Int64(Some(-1)),
    1741
    0
                DataType::Float16 => ScalarValue::Float16(Some(f16::from_f32(-1.0))),
    1742
    0
                DataType::Float32 => ScalarValue::Float32(Some(-1.0)),
    1743
    0
                DataType::Float64 => ScalarValue::Float64(Some(-1.0)),
    1744
    0
                DataType::Decimal32(precision, scale) => {
    1745
    0
                    Self::validate_decimal_or_internal_err::<Decimal32Type>(
    1746
    0
                        *precision, *scale,
    1747
    0
                    )?;
    1748
    0
                    assert_or_internal_err!(*scale >= 0, "Negative scale is not supported");
    1749
    0
                    match 10_i32.checked_pow(*scale as u32) {
    1750
    0
                        Some(value) => {
    1751
    0
                            ScalarValue::Decimal32(Some(-value), *precision, *scale)
    1752
                        }
    1753
    0
                        None => return _internal_err!("Unsupported scale {scale}"),
    1754
                    }
    1755
                }
    1756
    0
                DataType::Decimal64(precision, scale) => {
    1757
    0
                    Self::validate_decimal_or_internal_err::<Decimal64Type>(
    1758
    0
                        *precision, *scale,
    1759
    0
                    )?;
    1760
    0
                    assert_or_internal_err!(*scale >= 0, "Negative scale is not supported");
    1761
    0
                    match i64::from(10).checked_pow(*scale as u32) {
    1762
    0
                        Some(value) => {
    1763
    0
                            ScalarValue::Decimal64(Some(-value), *precision, *scale)
    1764
                        }
    1765
    0
                        None => return _internal_err!("Unsupported scale {scale}"),
    1766
                    }
    1767
                }
    1768
    0
                DataType::Decimal128(precision, scale) => {
    1769
    0
                    Self::validate_decimal_or_internal_err::<Decimal128Type>(
    1770
    0
                        *precision, *scale,
    1771
    0
                    )?;
    1772
    0
                    assert_or_internal_err!(*scale >= 0, "Negative scale is not supported");
    1773
    0
                    match i128::from(10).checked_pow(*scale as u32) {
    1774
    0
                        Some(value) => {
    1775
    0
                            ScalarValue::Decimal128(Some(-value), *precision, *scale)
    1776
                        }
    1777
    0
                        None => return _internal_err!("Unsupported scale {scale}"),
    1778
                    }
    1779
                }
    1780
    0
                DataType::Decimal256(precision, scale) => {
    1781
    0
                    Self::validate_decimal_or_internal_err::<Decimal256Type>(
    1782
    0
                        *precision, *scale,
    1783
    0
                    )?;
    1784
    0
                    assert_or_internal_err!(*scale >= 0, "Negative scale is not supported");
    1785
    0
                    match i256::from(10).checked_pow(*scale as u32) {
    1786
    0
                        Some(value) => {
    1787
    0
                            ScalarValue::Decimal256(Some(-value), *precision, *scale)
    1788
                        }
    1789
    0
                        None => return _internal_err!("Unsupported scale {scale}"),
    1790
                    }
    1791
                }
    1792
                _ => {
    1793
    0
                    return _not_impl_err!(
    1794
    0
                        "Can't create a negative one scalar from data_type \"{datatype}\""
    1795
                    );
    1796
                }
    1797
            })
    1798
    0
        }
    1799
    1800
    0
        pub fn new_ten(datatype: &DataType) -> Result<ScalarValue> {
    1801
    0
            Ok(match datatype {
    1802
    0
                DataType::Int8 => ScalarValue::Int8(Some(10)),
    1803
    0
                DataType::Int16 => ScalarValue::Int16(Some(10)),
    1804
    0
                DataType::Int32 => ScalarValue::Int32(Some(10)),
    1805
    0
                DataType::Int64 => ScalarValue::Int64(Some(10)),
    1806
    0
                DataType::UInt8 => ScalarValue::UInt8(Some(10)),
    1807
    0
                DataType::UInt16 => ScalarValue::UInt16(Some(10)),
    1808
    0
                DataType::UInt32 => ScalarValue::UInt32(Some(10)),
    1809
    0
                DataType::UInt64 => ScalarValue::UInt64(Some(10)),
    1810
    0
                DataType::Float16 => ScalarValue::Float16(Some(f16::from_f32(10.0))),
    1811
    0
                DataType::Float32 => ScalarValue::Float32(Some(10.0)),
    1812
    0
                DataType::Float64 => ScalarValue::Float64(Some(10.0)),
    1813
    0
                DataType::Decimal32(precision, scale) => {
    1814
    0
                    Self::validate_decimal_or_internal_err::<Decimal32Type>(
    1815
    0
                        *precision, *scale,
    1816
    0
                    )?;
    1817
    0
                    assert_or_internal_err!(*scale >= 0, "Negative scale is not supported");
    1818
    0
                    match 10_i32.checked_pow((*scale + 1) as u32) {
    1819
    0
                        Some(value) => {
    1820
    0
                            ScalarValue::Decimal32(Some(value), *precision, *scale)
    1821
                        }
    1822
    0
                        None => return _internal_err!("Unsupported scale {scale}"),
    1823
                    }
    1824
                }
    1825
    0
                DataType::Decimal64(precision, scale) => {
    1826
    0
                    Self::validate_decimal_or_internal_err::<Decimal64Type>(
    1827
    0
                        *precision, *scale,
    1828
    0
                    )?;
    1829
    0
                    assert_or_internal_err!(*scale >= 0, "Negative scale is not supported");
    1830
    0
                    match i64::from(10).checked_pow((*scale + 1) as u32) {
    1831
    0
                        Some(value) => {
    1832
    0
                            ScalarValue::Decimal64(Some(value), *precision, *scale)
    1833
                        }
    1834
    0
                        None => return _internal_err!("Unsupported scale {scale}"),
    1835
                    }
    1836
                }
    1837
    0
                DataType::Decimal128(precision, scale) => {
    1838
    0
                    Self::validate_decimal_or_internal_err::<Decimal128Type>(
    1839
    0
                        *precision, *scale,
    1840
    0
                    )?;
    1841
    0
                    assert_or_internal_err!(*scale >= 0, "Negative scale is not supported");
    1842
    0
                    match i128::from(10).checked_pow((*scale + 1) as u32) {
    1843
    0
                        Some(value) => {
    1844
    0
                            ScalarValue::Decimal128(Some(value), *precision, *scale)
    1845
                        }
    1846
    0
                        None => return _internal_err!("Unsupported scale {scale}"),
    1847
                    }
    1848
                }
    1849
    0
                DataType::Decimal256(precision, scale) => {
    1850
    0
                    Self::validate_decimal_or_internal_err::<Decimal256Type>(
    1851
    0
                        *precision, *scale,
    1852
    0
                    )?;
    1853
    0
                    assert_or_internal_err!(*scale >= 0, "Negative scale is not supported");
    1854
    0
                    match i256::from(10).checked_pow((*scale + 1) as u32) {
    1855
    0
                        Some(value) => {
    1856
    0
                            ScalarValue::Decimal256(Some(value), *precision, *scale)
    1857
                        }
    1858
    0
                        None => return _internal_err!("Unsupported scale {scale}"),
    1859
                    }
    1860
                }
    1861
                _ => {
    1862
    0
                    return _not_impl_err!(
    1863
    0
                        "Can't create a ten scalar from data_type \"{datatype}\""
    1864
                    );
    1865
                }
    1866
            })
    1867
    0
        }
    1868
    1869
        /// return the [`DataType`] of this `ScalarValue`
    1870
    2.81k
        pub fn data_type(&self) -> DataType {
    1871
    2.81k
            match self {
    1872
    20
                ScalarValue::Boolean(_) => DataType::Boolean,
    1873
    36
                ScalarValue::UInt8(_) => DataType::UInt8,
    1874
    36
                ScalarValue::UInt16(_) => DataType::UInt16,
    1875
    36
                ScalarValue::UInt32(_) => DataType::UInt32,
    1876
    36
                ScalarValue::UInt64(_) => DataType::UInt64,
    1877
    36
                ScalarValue::Int8(_) => DataType::Int8,
    1878
    36
                ScalarValue::Int16(_) => DataType::Int16,
    1879
    1.65k
                ScalarValue::Int32(_) => DataType::Int32,
    1880
    148
                ScalarValue::Int64(_) => DataType::Int64,
    1881
    0
                ScalarValue::Decimal32(_, precision, scale) => {
    1882
    0
                    DataType::Decimal32(*precision, *scale)
    1883
                }
    1884
    0
                ScalarValue::Decimal64(_, precision, scale) => {
    1885
    0
                    DataType::Decimal64(*precision, *scale)
    1886
                }
    1887
    8
                ScalarValue::Decimal128(_, precision, scale) => {
    1888
    8
                    DataType::Decimal128(*precision, *scale)
    1889
                }
    1890
    0
                ScalarValue::Decimal256(_, precision, scale) => {
    1891
    0
                    DataType::Decimal256(*precision, *scale)
    1892
                }
    1893
    0
                ScalarValue::TimestampSecond(_, tz_opt) => {
    1894
    0
                    DataType::Timestamp(TimeUnit::Second, tz_opt.clone())
    1895
                }
    1896
    0
                ScalarValue::TimestampMillisecond(_, tz_opt) => {
    1897
    0
                    DataType::Timestamp(TimeUnit::Millisecond, tz_opt.clone())
    1898
                }
    1899
    23
                ScalarValue::TimestampMicrosecond(_, tz_opt) => {
    1900
    23
                    DataType::Timestamp(TimeUnit::Microsecond, tz_opt.clone())
    1901
                }
    1902
    0
                ScalarValue::TimestampNanosecond(_, tz_opt) => {
    1903
    0
                    DataType::Timestamp(TimeUnit::Nanosecond, tz_opt.clone())
    1904
                }
    1905
    0
                ScalarValue::Float16(_) => DataType::Float16,
    1906
    13
                ScalarValue::Float32(_) => DataType::Float32,
    1907
    88
                ScalarValue::Float64(_) => DataType::Float64,
    1908
    212
                ScalarValue::Utf8(_) => DataType::Utf8,
    1909
    12
                ScalarValue::LargeUtf8(_) => DataType::LargeUtf8,
    1910
    12
                ScalarValue::Utf8View(_) => DataType::Utf8View,
    1911
    95
                ScalarValue::Binary(_) => DataType::Binary,
    1912
    11
                ScalarValue::BinaryView(_) => DataType::BinaryView,
    1913
    0
                ScalarValue::FixedSizeBinary(sz, _) => DataType::FixedSizeBinary(*sz),
    1914
    11
                ScalarValue::LargeBinary(_) => DataType::LargeBinary,
    1915
    0
                ScalarValue::List(arr) => arr.data_type().to_owned(),
    1916
    0
                ScalarValue::LargeList(arr) => arr.data_type().to_owned(),
    1917
    0
                ScalarValue::FixedSizeList(arr) => arr.data_type().to_owned(),
    1918
    85
                ScalarValue::Struct(arr) => arr.data_type().to_owned(),
    1919
    0
                ScalarValue::Map(arr) => arr.data_type().to_owned(),
    1920
    36
                ScalarValue::Date32(_) => DataType::Date32,
    1921
    36
                ScalarValue::Date64(_) => DataType::Date64,
    1922
    0
                ScalarValue::Time32Second(_) => DataType::Time32(TimeUnit::Second),
    1923
    0
                ScalarValue::Time32Millisecond(_) => DataType::Time32(TimeUnit::Millisecond),
    1924
    0
                ScalarValue::Time64Microsecond(_) => DataType::Time64(TimeUnit::Microsecond),
    1925
    0
                ScalarValue::Time64Nanosecond(_) => DataType::Time64(TimeUnit::Nanosecond),
    1926
                ScalarValue::IntervalYearMonth(_) => {
    1927
    0
                    DataType::Interval(IntervalUnit::YearMonth)
    1928
                }
    1929
    0
                ScalarValue::IntervalDayTime(_) => DataType::Interval(IntervalUnit::DayTime),
    1930
                ScalarValue::IntervalMonthDayNano(_) => {
    1931
    0
                    DataType::Interval(IntervalUnit::MonthDayNano)
    1932
                }
    1933
    0
                ScalarValue::DurationSecond(_) => DataType::Duration(TimeUnit::Second),
    1934
                ScalarValue::DurationMillisecond(_) => {
    1935
    0
                    DataType::Duration(TimeUnit::Millisecond)
    1936
                }
    1937
                ScalarValue::DurationMicrosecond(_) => {
    1938
    0
                    DataType::Duration(TimeUnit::Microsecond)
    1939
                }
    1940
                ScalarValue::DurationNanosecond(_) => {
    1941
    0
                    DataType::Duration(TimeUnit::Nanosecond)
    1942
                }
    1943
    0
                ScalarValue::Union(_, fields, mode) => DataType::Union(fields.clone(), *mode),
    1944
    19
                ScalarValue::Dictionary(k, v) => {
    1945
    19
                    DataType::Dictionary(k.clone(), Box::new(v.data_type()))
    1946
                }
    1947
    114
                ScalarValue::Null => DataType::Null,
    1948
            }
    1949
    2.81k
        }
    1950
    1951
        /// Calculate arithmetic negation for a scalar value
    1952
    0
        pub fn arithmetic_negate(&self) -> Result<Self> {
    1953
    0
            fn neg_checked_with_ctx<T: ArrowNativeTypeOp>(
    1954
    0
                v: T,
    1955
    0
                ctx: impl Fn() -> String,
    1956
    0
            ) -> Result<T> {
    1957
    0
                v.neg_checked()
    1958
    0
                    .map_err(|e| arrow_datafusion_err!(e).context(ctx()))
    1959
    0
            }
    1960
    0
            match self {
    1961
                ScalarValue::Int8(None)
    1962
                | ScalarValue::Int16(None)
    1963
                | ScalarValue::Int32(None)
    1964
                | ScalarValue::Int64(None)
    1965
                | ScalarValue::Float16(None)
    1966
                | ScalarValue::Float32(None)
    1967
    0
                | ScalarValue::Float64(None) => Ok(self.clone()),
    1968
    0
                ScalarValue::Float16(Some(v)) => {
    1969
    0
                    Ok(ScalarValue::Float16(Some(f16::from_f32(-v.to_f32()))))
    1970
                }
    1971
    0
                ScalarValue::Float64(Some(v)) => Ok(ScalarValue::Float64(Some(-v))),
    1972
    0
                ScalarValue::Float32(Some(v)) => Ok(ScalarValue::Float32(Some(-v))),
    1973
    0
                ScalarValue::Int8(Some(v)) => Ok(ScalarValue::Int8(Some(v.neg_checked()?))),
    1974
    0
                ScalarValue::Int16(Some(v)) => Ok(ScalarValue::Int16(Some(v.neg_checked()?))),
    1975
    0
                ScalarValue::Int32(Some(v)) => Ok(ScalarValue::Int32(Some(v.neg_checked()?))),
    1976
    0
                ScalarValue::Int64(Some(v)) => Ok(ScalarValue::Int64(Some(v.neg_checked()?))),
    1977
    0
                ScalarValue::IntervalYearMonth(Some(v)) => Ok(
    1978
    0
                    ScalarValue::IntervalYearMonth(Some(neg_checked_with_ctx(*v, || {
    1979
    0
                        format!("In negation of IntervalYearMonth({v})")
    1980
    0
                    })?)),
    1981
                ),
    1982
    0
                ScalarValue::IntervalDayTime(Some(v)) => {
    1983
    0
                    let (days, ms) = IntervalDayTimeType::to_parts(*v);
    1984
    0
                    let val = IntervalDayTimeType::make_value(
    1985
    0
                        neg_checked_with_ctx(days, || {
    1986
    0
                            format!("In negation of days {days} in IntervalDayTime")
    1987
    0
                        })?,
    1988
    0
                        neg_checked_with_ctx(ms, || {
    1989
    0
                            format!("In negation of milliseconds {ms} in IntervalDayTime")
    1990
    0
                        })?,
    1991
                    );
    1992
    0
                    Ok(ScalarValue::IntervalDayTime(Some(val)))
    1993
                }
    1994
    0
                ScalarValue::IntervalMonthDayNano(Some(v)) => {
    1995
    0
                    let (months, days, nanos) = IntervalMonthDayNanoType::to_parts(*v);
    1996
    0
                    let val = IntervalMonthDayNanoType::make_value(
    1997
    0
                        neg_checked_with_ctx(months, || {
    1998
    0
                            format!("In negation of months {months} of IntervalMonthDayNano")
    1999
    0
                        })?,
    2000
    0
                        neg_checked_with_ctx(days, || {
    2001
    0
                            format!("In negation of days {days} of IntervalMonthDayNano")
    2002
    0
                        })?,
    2003
    0
                        neg_checked_with_ctx(nanos, || {
    2004
    0
                            format!("In negation of nanos {nanos} of IntervalMonthDayNano")
    2005
    0
                        })?,
    2006
                    );
    2007
    0
                    Ok(ScalarValue::IntervalMonthDayNano(Some(val)))
    2008
                }
    2009
    0
                ScalarValue::Decimal32(Some(v), precision, scale) => {
    2010
                    Ok(ScalarValue::Decimal32(
    2011
    0
                        Some(neg_checked_with_ctx(*v, || {
    2012
    0
                            format!("In negation of Decimal32({v}, {precision}, {scale})")
    2013
    0
                        })?),
    2014
    0
                        *precision,
    2015
    0
                        *scale,
    2016
                    ))
    2017
                }
    2018
    0
                ScalarValue::Decimal64(Some(v), precision, scale) => {
    2019
                    Ok(ScalarValue::Decimal64(
    2020
    0
                        Some(neg_checked_with_ctx(*v, || {
    2021
    0
                            format!("In negation of Decimal64({v}, {precision}, {scale})")
    2022
    0
                        })?),
    2023
    0
                        *precision,
    2024
    0
                        *scale,
    2025
                    ))
    2026
                }
    2027
    0
                ScalarValue::Decimal128(Some(v), precision, scale) => {
    2028
                    Ok(ScalarValue::Decimal128(
    2029
    0
                        Some(neg_checked_with_ctx(*v, || {
    2030
    0
                            format!("In negation of Decimal128({v}, {precision}, {scale})")
    2031
    0
                        })?),
    2032
    0
                        *precision,
    2033
    0
                        *scale,
    2034
                    ))
    2035
                }
    2036
    0
                ScalarValue::Decimal256(Some(v), precision, scale) => {
    2037
                    Ok(ScalarValue::Decimal256(
    2038
    0
                        Some(neg_checked_with_ctx(*v, || {
    2039
    0
                            format!("In negation of Decimal256({v}, {precision}, {scale})")
    2040
    0
                        })?),
    2041
    0
                        *precision,
    2042
    0
                        *scale,
    2043
                    ))
    2044
                }
    2045
    0
                ScalarValue::TimestampSecond(Some(v), tz) => {
    2046
                    Ok(ScalarValue::TimestampSecond(
    2047
    0
                        Some(neg_checked_with_ctx(*v, || {
    2048
    0
                            format!("In negation of TimestampSecond({v})")
    2049
    0
                        })?),
    2050
    0
                        tz.clone(),
    2051
                    ))
    2052
                }
    2053
    0
                ScalarValue::TimestampNanosecond(Some(v), tz) => {
    2054
                    Ok(ScalarValue::TimestampNanosecond(
    2055
    0
                        Some(neg_checked_with_ctx(*v, || {
    2056
    0
                            format!("In negation of TimestampNanoSecond({v})")
    2057
    0
                        })?),
    2058
    0
                        tz.clone(),
    2059
                    ))
    2060
                }
    2061
    0
                ScalarValue::TimestampMicrosecond(Some(v), tz) => {
    2062
                    Ok(ScalarValue::TimestampMicrosecond(
    2063
    0
                        Some(neg_checked_with_ctx(*v, || {
    2064
    0
                            format!("In negation of TimestampMicroSecond({v})")
    2065
    0
                        })?),
    2066
    0
                        tz.clone(),
    2067
                    ))
    2068
                }
    2069
    0
                ScalarValue::TimestampMillisecond(Some(v), tz) => {
    2070
                    Ok(ScalarValue::TimestampMillisecond(
    2071
    0
                        Some(neg_checked_with_ctx(*v, || {
    2072
    0
                            format!("In negation of TimestampMilliSecond({v})")
    2073
    0
                        })?),
    2074
    0
                        tz.clone(),
    2075
                    ))
    2076
                }
    2077
    0
                value => _internal_err!(
    2078
    0
                    "Can not run arithmetic negative on scalar value {value:?}"
    2079
                ),
    2080
            }
    2081
    0
        }
    2082
    2083
        /// Wrapping addition of `ScalarValue`
    2084
        ///
    2085
        /// NB: operating on `ScalarValue` directly is not efficient, performance sensitive code
    2086
        /// should operate on Arrays directly, using vectorized array kernels
    2087
    0
        pub fn add<T: Borrow<ScalarValue>>(&self, other: T) -> Result<ScalarValue> {
    2088
    0
            let r = add_wrapping(&self.to_scalar()?, &other.borrow().to_scalar()?)?;
    2089
    0
            Self::try_from_array(r.as_ref(), 0)
    2090
    0
        }
    2091
        /// Checked addition of `ScalarValue`
    2092
        ///
    2093
        /// NB: operating on `ScalarValue` directly is not efficient, performance sensitive code
    2094
        /// should operate on Arrays directly, using vectorized array kernels
    2095
    0
        pub fn add_checked<T: Borrow<ScalarValue>>(&self, other: T) -> Result<ScalarValue> {
    2096
    0
            let r = add(&self.to_scalar()?, &other.borrow().to_scalar()?)?;
    2097
    0
            Self::try_from_array(r.as_ref(), 0)
    2098
    0
        }
    2099
    2100
        /// Wrapping subtraction of `ScalarValue`
    2101
        ///
    2102
        /// NB: operating on `ScalarValue` directly is not efficient, performance sensitive code
    2103
        /// should operate on Arrays directly, using vectorized array kernels
    2104
    0
        pub fn sub<T: Borrow<ScalarValue>>(&self, other: T) -> Result<ScalarValue> {
    2105
    0
            let r = sub_wrapping(&self.to_scalar()?, &other.borrow().to_scalar()?)?;
    2106
    0
            Self::try_from_array(r.as_ref(), 0)
    2107
    0
        }
    2108
    2109
        /// Checked subtraction of `ScalarValue`
    2110
        ///
    2111
        /// NB: operating on `ScalarValue` directly is not efficient, performance sensitive code
    2112
        /// should operate on Arrays directly, using vectorized array kernels
    2113
    0
        pub fn sub_checked<T: Borrow<ScalarValue>>(&self, other: T) -> Result<ScalarValue> {
    2114
    0
            let r = sub(&self.to_scalar()?, &other.borrow().to_scalar()?)?;
    2115
    0
            Self::try_from_array(r.as_ref(), 0)
    2116
    0
        }
    2117
    2118
        /// Wrapping multiplication of `ScalarValue`
    2119
        ///
    2120
        /// NB: operating on `ScalarValue` directly is not efficient, performance sensitive code
    2121
        /// should operate on Arrays directly, using vectorized array kernels.
    2122
    0
        pub fn mul<T: Borrow<ScalarValue>>(&self, other: T) -> Result<ScalarValue> {
    2123
    0
            let r = mul_wrapping(&self.to_scalar()?, &other.borrow().to_scalar()?)?;
    2124
    0
            Self::try_from_array(r.as_ref(), 0)
    2125
    0
        }
    2126
    2127
        /// Checked multiplication of `ScalarValue`
    2128
        ///
    2129
        /// NB: operating on `ScalarValue` directly is not efficient, performance sensitive code
    2130
        /// should operate on Arrays directly, using vectorized array kernels.
    2131
    0
        pub fn mul_checked<T: Borrow<ScalarValue>>(&self, other: T) -> Result<ScalarValue> {
    2132
    0
            let r = mul(&self.to_scalar()?, &other.borrow().to_scalar()?)?;
    2133
    0
            Self::try_from_array(r.as_ref(), 0)
    2134
    0
        }
    2135
    2136
        /// Performs `lhs / rhs`
    2137
        ///
    2138
        /// Overflow or division by zero will result in an error, with exception to
    2139
        /// floating point numbers, which instead follow the IEEE 754 rules.
    2140
        ///
    2141
        /// NB: operating on `ScalarValue` directly is not efficient, performance sensitive code
    2142
        /// should operate on Arrays directly, using vectorized array kernels.
    2143
    0
        pub fn div<T: Borrow<ScalarValue>>(&self, other: T) -> Result<ScalarValue> {
    2144
    0
            let r = div(&self.to_scalar()?, &other.borrow().to_scalar()?)?;
    2145
    0
            Self::try_from_array(r.as_ref(), 0)
    2146
    0
        }
    2147
    2148
        /// Performs `lhs % rhs`
    2149
        ///
    2150
        /// Overflow or division by zero will result in an error, with exception to
    2151
        /// floating point numbers, which instead follow the IEEE 754 rules.
    2152
        ///
    2153
        /// NB: operating on `ScalarValue` directly is not efficient, performance sensitive code
    2154
        /// should operate on Arrays directly, using vectorized array kernels.
    2155
    0
        pub fn rem<T: Borrow<ScalarValue>>(&self, other: T) -> Result<ScalarValue> {
    2156
    0
            let r = rem(&self.to_scalar()?, &other.borrow().to_scalar()?)?;
    2157
    0
            Self::try_from_array(r.as_ref(), 0)
    2158
    0
        }
    2159
    2160
    0
        pub fn is_unsigned(&self) -> bool {
    2161
    0
            matches!(
    2162
    0
                self,
    2163
                ScalarValue::UInt8(_)
    2164
                    | ScalarValue::UInt16(_)
    2165
                    | ScalarValue::UInt32(_)
    2166
                    | ScalarValue::UInt64(_)
    2167
            )
    2168
    0
        }
    2169
    2170
        /// whether this value is null or not.
    2171
    594
        pub fn is_null(&self) -> bool {
    2172
    594
            match self {
    2173
    4
                ScalarValue::Boolean(v) => v.is_none(),
    2174
    30
                ScalarValue::Null => true,
    2175
    0
                ScalarValue::Float16(v) => v.is_none(),
    2176
    3
                ScalarValue::Float32(v) => v.is_none(),
    2177
    20
                ScalarValue::Float64(v) => v.is_none(),
    2178
    0
                ScalarValue::Decimal32(v, _, _) => v.is_none(),
    2179
    0
                ScalarValue::Decimal64(v, _, _) => v.is_none(),
    2180
    0
                ScalarValue::Decimal128(v, _, _) => v.is_none(),
    2181
    0
                ScalarValue::Decimal256(v, _, _) => v.is_none(),
    2182
    8
                ScalarValue::Int8(v) => v.is_none(),
    2183
    8
                ScalarValue::Int16(v) => v.is_none(),
    2184
    298
                ScalarValue::Int32(v) => v.is_none(),
    2185
    31
                ScalarValue::Int64(v) => v.is_none(),
    2186
    8
                ScalarValue::UInt8(v) => v.is_none(),
    2187
    8
                ScalarValue::UInt16(v) => v.is_none(),
    2188
    8
                ScalarValue::UInt32(v) => v.is_none(),
    2189
    8
                ScalarValue::UInt64(v) => v.is_none(),
    2190
    68
                ScalarValue::Utf8(v)
    2191
    2
                | ScalarValue::Utf8View(v)
    2192
    72
                | ScalarValue::LargeUtf8(v2) => v.is_none(),
    2193
    13
                ScalarValue::Binary(v)
    2194
    1
                | ScalarValue::BinaryView(v)
    2195
    0
                | ScalarValue::FixedSizeBinary(_, v)
    2196
    15
                | ScalarValue::LargeBinary(v1) => v.is_none(),
    2197
                // arr.len() should be 1 for a list scalar, but we don't seem to
    2198
                // enforce that anywhere, so we still check against array length.
    2199
    0
                ScalarValue::List(arr) => arr.len() == arr.null_count(),
    2200
    0
                ScalarValue::LargeList(arr) => arr.len() == arr.null_count(),
    2201
    0
                ScalarValue::FixedSizeList(arr) => arr.len() == arr.null_count(),
    2202
    47
                ScalarValue::Struct(arr) => arr.len() == arr.null_count(),
    2203
    0
                ScalarValue::Map(arr) => arr.len() == arr.null_count(),
    2204
    8
                ScalarValue::Date32(v) => v.is_none(),
    2205
    8
                ScalarValue::Date64(v) => v.is_none(),
    2206
    0
                ScalarValue::Time32Second(v) => v.is_none(),
    2207
    0
                ScalarValue::Time32Millisecond(v) => v.is_none(),
    2208
    0
                ScalarValue::Time64Microsecond(v) => v.is_none(),
    2209
    0
                ScalarValue::Time64Nanosecond(v) => v.is_none(),
    2210
    0
                ScalarValue::TimestampSecond(v, _) => v.is_none(),
    2211
    0
                ScalarValue::TimestampMillisecond(v, _) => v.is_none(),
    2212
    3
                ScalarValue::TimestampMicrosecond(v, _) => v.is_none(),
    2213
    0
                ScalarValue::TimestampNanosecond(v, _) => v.is_none(),
    2214
    0
                ScalarValue::IntervalYearMonth(v) => v.is_none(),
    2215
    0
                ScalarValue::IntervalDayTime(v) => v.is_none(),
    2216
    0
                ScalarValue::IntervalMonthDayNano(v) => v.is_none(),
    2217
    0
                ScalarValue::DurationSecond(v) => v.is_none(),
    2218
    0
                ScalarValue::DurationMillisecond(v) => v.is_none(),
    2219
    0
                ScalarValue::DurationMicrosecond(v) => v.is_none(),
    2220
    0
                ScalarValue::DurationNanosecond(v) => v.is_none(),
    2221
    0
                ScalarValue::Union(v, _, _) => match v {
    2222
    0
                    Some((_, s)) => s.is_null(),
    2223
    0
                    None => true,
    2224
                },
    2225
    7
                ScalarValue::Dictionary(_, v) => v.is_null(),
    2226
            }
    2227
    594
        }
    2228
    2229
        /// Absolute distance between two numeric values (of the same type). This method will return
    2230
        /// None if either one of the arguments are null. It might also return None if the resulting
    2231
        /// distance is greater than [`usize::MAX`]. If the type is a float, then the distance will be
    2232
        /// rounded to the nearest integer.
    2233
        ///
    2234
        ///
    2235
        /// Note: the datatype itself must support subtraction.
    2236
    0
        pub fn distance(&self, other: &ScalarValue) -> Option<usize> {
    2237
    0
            match (self, other) {
    2238
    0
                (Self::Int8(Some(l)), Self::Int8(Some(r))) => Some(l.abs_diff(*r) as _),
    2239
    0
                (Self::Int16(Some(l)), Self::Int16(Some(r))) => Some(l.abs_diff(*r) as _),
    2240
    0
                (Self::Int32(Some(l)), Self::Int32(Some(r))) => Some(l.abs_diff(*r) as _),
    2241
    0
                (Self::Int64(Some(l)), Self::Int64(Some(r))) => Some(l.abs_diff(*r) as _),
    2242
    0
                (Self::UInt8(Some(l)), Self::UInt8(Some(r))) => Some(l.abs_diff(*r) as _),
    2243
    0
                (Self::UInt16(Some(l)), Self::UInt16(Some(r))) => Some(l.abs_diff(*r) as _),
    2244
    0
                (Self::UInt32(Some(l)), Self::UInt32(Some(r))) => Some(l.abs_diff(*r) as _),
    2245
    0
                (Self::UInt64(Some(l)), Self::UInt64(Some(r))) => Some(l.abs_diff(*r) as _),
    2246
                // TODO: we might want to look into supporting ceil/floor here for floats.
    2247
    0
                (Self::Float16(Some(l)), Self::Float16(Some(r))) => {
    2248
    0
                    Some((f16::to_f32(*l) - f16::to_f32(*r)).abs().round() as _)
    2249
                }
    2250
    0
                (Self::Float32(Some(l)), Self::Float32(Some(r))) => {
    2251
    0
                    Some((l - r).abs().round() as _)
    2252
                }
    2253
    0
                (Self::Float64(Some(l)), Self::Float64(Some(r))) => {
    2254
    0
                    Some((l - r).abs().round() as _)
    2255
                }
    2256
                (
    2257
    0
                    Self::Decimal128(Some(l), lprecision, lscale),
    2258
    0
                    Self::Decimal128(Some(r), rprecision, rscale),
    2259
                ) => {
    2260
    0
                    if lprecision == rprecision && lscale == rscale {
    2261
    0
                        l.checked_sub(*r)?.checked_abs()?.to_usize()
    2262
                    } else {
    2263
    0
                        None
    2264
                    }
    2265
                }
    2266
                (
    2267
    0
                    Self::Decimal256(Some(l), lprecision, lscale),
    2268
    0
                    Self::Decimal256(Some(r), rprecision, rscale),
    2269
                ) => {
    2270
    0
                    if lprecision == rprecision && lscale == rscale {
    2271
    0
                        l.checked_sub(*r)?.checked_abs()?.to_usize()
    2272
                    } else {
    2273
    0
                        None
    2274
                    }
    2275
                }
    2276
    0
                _ => None,
    2277
            }
    2278
    0
        }
    2279
    2280
        /// Converts a scalar value into an 1-row array.
    2281
        ///
    2282
        /// # Errors
    2283
        ///
    2284
        /// Errors if the ScalarValue cannot be converted into a 1-row array
    2285
    537
        pub fn to_array(&self) -> Result<ArrayRef> {
    2286
    537
            self.to_array_of_size(1)
    2287
    537
        }
    2288
    2289
        /// Converts a scalar into an arrow [`Scalar`] (which implements
    2290
        /// the [`Datum`] interface).
    2291
        ///
    2292
        /// This can be used to call arrow compute kernels such as `lt`
    2293
        ///
    2294
        /// # Errors
    2295
        ///
    2296
        /// Errors if the ScalarValue cannot be converted into a 1-row array
    2297
        ///
    2298
        /// # Example
    2299
        /// ```
    2300
        /// use arrow::array::{BooleanArray, Int32Array};
    2301
        /// use datafusion_common::ScalarValue;
    2302
        ///
    2303
        /// let arr = Int32Array::from(vec![Some(1), None, Some(10)]);
    2304
        /// let five = ScalarValue::Int32(Some(5));
    2305
        ///
    2306
        /// let result =
    2307
        ///     arrow::compute::kernels::cmp::lt(&arr, &five.to_scalar().unwrap()).unwrap();
    2308
        ///
    2309
        /// let expected = BooleanArray::from(vec![Some(true), None, Some(false)]);
    2310
        ///
    2311
        /// assert_eq!(&result, &expected);
    2312
        /// ```
    2313
        /// [`Datum`]: arrow::array::Datum
    2314
    0
        pub fn to_scalar(&self) -> Result<Scalar<ArrayRef>> {
    2315
    0
            Ok(Scalar::new(self.to_array_of_size(1)?))
    2316
    0
        }
    2317
    2318
        /// Converts an iterator of references [`ScalarValue`] into an [`ArrayRef`]
    2319
        /// corresponding to those values. For example, an iterator of
    2320
        /// [`ScalarValue::Int32`] would be converted to an [`Int32Array`].
    2321
        ///
    2322
        /// Returns an error if the iterator is empty or if the
    2323
        /// [`ScalarValue`]s are not all the same type
    2324
        ///
    2325
        /// # Example
    2326
        /// ```
    2327
        /// use arrow::array::{ArrayRef, BooleanArray};
    2328
        /// use datafusion_common::ScalarValue;
    2329
        ///
    2330
        /// let scalars = vec![
    2331
        ///     ScalarValue::Boolean(Some(true)),
    2332
        ///     ScalarValue::Boolean(None),
    2333
        ///     ScalarValue::Boolean(Some(false)),
    2334
        /// ];
    2335
        ///
    2336
        /// // Build an Array from the list of ScalarValues
    2337
        /// let array = ScalarValue::iter_to_array(scalars.into_iter()).unwrap();
    2338
        ///
    2339
        /// let expected: ArrayRef =
    2340
        ///     std::sync::Arc::new(BooleanArray::from(vec![Some(true), None, Some(false)]));
    2341
        ///
    2342
        /// assert_eq!(&array, &expected);
    2343
        /// ```
    2344
    159
        pub fn iter_to_array(
    2345
    159
            scalars: impl IntoIterator<Item = ScalarValue>,
    2346
    159
        ) -> Result<ArrayRef> {
    2347
    159
            let mut scalars = scalars.into_iter().peekable();
    2348
    2349
            // figure out the type based on the first element
    2350
    159
            let data_type = match scalars.peek() {
    2351
                None => {
    2352
    0
                    return _exec_err!("Empty iterator passed to ScalarValue::iter_to_array");
    2353
                }
    2354
    159
                Some(sv) => sv.data_type(),
    2355
            };
    2356
    2357
            /// Creates an array of $ARRAY_TY by unpacking values of
    2358
            /// SCALAR_TY for primitive types
    2359
            macro_rules! build_array_primitive {
    2360
                ($ARRAY_TY:ident, $SCALAR_TY:ident) => {{
    2361
                    {
    2362
    213
                        let array = scalars.map(|sv| {
    2363
    213
                            if let ScalarValue::$SCALAR_TY(v) = sv {
    2364
    213
                                Ok(v)
    2365
                            } else {
    2366
    0
                                _exec_err!(
    2367
    0
                                    "Inconsistent types in ScalarValue::iter_to_array. \
    2368
    0
                                        Expected {:?}, got {:?}",
    2369
                                    data_type, sv
    2370
                                )
    2371
                            }
    2372
    213
                        })
    2373
                        .collect::<Result<$ARRAY_TY>>()?;
    2374
                        Arc::new(array)
    2375
                    }
    2376
                }};
    2377
            }
    2378
    2379
            macro_rules! build_array_primitive_tz {
    2380
                ($ARRAY_TY:ident, $SCALAR_TY:ident, $TZ:expr) => {{
    2381
                    {
    2382
    6
                        let array = scalars.map(|sv| {
    2383
    6
                            if let ScalarValue::$SCALAR_TY(v, _) = sv {
    2384
    6
                                Ok(v)
    2385
                            } else {
    2386
    0
                                _exec_err!(
    2387
    0
                                    "Inconsistent types in ScalarValue::iter_to_array. \
    2388
    0
                                        Expected {:?}, got {:?}",
    2389
                                    data_type, sv
    2390
                                )
    2391
                            }
    2392
    6
                        })
    2393
                        .collect::<Result<$ARRAY_TY>>()?;
    2394
                        Arc::new(array.with_timezone_opt($TZ.clone()))
    2395
                    }
    2396
                }};
    2397
            }
    2398
    2399
            /// Creates an array of $ARRAY_TY by unpacking values of
    2400
            /// SCALAR_TY for "string-like" types.
    2401
            macro_rules! build_array_string {
    2402
                ($ARRAY_TY:ident, $SCALAR_TY:ident) => {{
    2403
                    {
    2404
    104
                        let array = scalars.map(|sv| {
    2405
    104
                            if let ScalarValue::$SCALAR_TY(v) = sv {
    2406
    104
                                Ok(v)
    2407
                            } else {
    2408
    0
                                _exec_err!(
    2409
    0
                                    "Inconsistent types in ScalarValue::iter_to_array. \
    2410
    0
                                        Expected {:?}, got {:?}",
    2411
                                    data_type, sv
    2412
                                )
    2413
                            }
    2414
    104
                        })
    2415
                        .collect::<Result<$ARRAY_TY>>()?;
    2416
                        Arc::new(array)
    2417
                    }
    2418
                }};
    2419
            }
    2420
    2421
    159
            let array: ArrayRef = match &data_type2 {
    2422
    0
                DataType::Decimal32(precision, scale) => {
    2423
    0
                    let decimal_array =
    2424
    0
                        ScalarValue::iter_to_decimal32_array(scalars, *precision, *scale)?;
    2425
    0
                    Arc::new(decimal_array)
    2426
                }
    2427
    0
                DataType::Decimal64(precision, scale) => {
    2428
    0
                    let decimal_array =
    2429
    0
                        ScalarValue::iter_to_decimal64_array(scalars, *precision, *scale)?;
    2430
    0
                    Arc::new(decimal_array)
    2431
                }
    2432
    8
                DataType::Decimal128(precision, scale) => {
    2433
    8
                    let decimal_array =
    2434
    8
                        ScalarValue::iter_to_decimal128_array(scalars, *precision, *scale)?0;
    2435
    8
                    Arc::new(decimal_array)
    2436
                }
    2437
    0
                DataType::Decimal256(precision, scale) => {
    2438
    0
                    let decimal_array =
    2439
    0
                        ScalarValue::iter_to_decimal256_array(scalars, *precision, *scale)?;
    2440
    0
                    Arc::new(decimal_array)
    2441
                }
    2442
    0
                DataType::Null => ScalarValue::iter_to_null_array(scalars)?,
    2443
    4
                DataType::Boolean => build_array_primitive!(BooleanArray, Boolean),
    2444
    0
                DataType::Float16 => build_array_primitive!(Float16Array, Float16),
    2445
    0
                DataType::Float32 => build_array_primitive!(Float32Array, Float32),
    2446
    8
                DataType::Float64 => build_array_primitive!(Float64Array, Float64),
    2447
    4
                DataType::Int8 => build_array_primitive!(Int8Array, Int8),
    2448
    4
                DataType::Int16 => build_array_primitive!(Int16Array, Int16),
    2449
    25
                DataType::Int32 => build_array_primitive!(Int32Array, Int32),
    2450
    19
                DataType::Int64 => build_array_primitive!(Int64Array, Int64),
    2451
    4
                DataType::UInt8 => build_array_primitive!(UInt8Array, UInt8),
    2452
    4
                DataType::UInt16 => build_array_primitive!(UInt16Array, UInt16),
    2453
    4
                DataType::UInt32 => build_array_primitive!(UInt32Array, UInt32),
    2454
    4
                DataType::UInt64 => build_array_primitive!(UInt64Array, UInt64),
    2455
    4
                DataType::Utf8View => build_array_string!(StringViewArray, Utf8View),
    2456
    22
                DataType::Utf8 => build_array_string!(StringArray, Utf8),
    2457
    4
                DataType::LargeUtf8 => build_array_string!(LargeStringArray, LargeUtf8),
    2458
    4
                DataType::BinaryView => build_array_string!(BinaryViewArray, BinaryView),
    2459
    4
                DataType::Binary => build_array_string!(BinaryArray, Binary),
    2460
    4
                DataType::LargeBinary => build_array_string!(LargeBinaryArray, LargeBinary),
    2461
    4
                DataType::Date32 => build_array_primitive!(Date32Array, Date32),
    2462
    4
                DataType::Date64 => build_array_primitive!(Date64Array, Date64),
    2463
                DataType::Time32(TimeUnit::Second) => {
    2464
    0
                    build_array_primitive!(Time32SecondArray, Time32Second)
    2465
                }
    2466
                DataType::Time32(TimeUnit::Millisecond) => {
    2467
    0
                    build_array_primitive!(Time32MillisecondArray, Time32Millisecond)
    2468
                }
    2469
                DataType::Time64(TimeUnit::Microsecond) => {
    2470
    0
                    build_array_primitive!(Time64MicrosecondArray, Time64Microsecond)
    2471
                }
    2472
                DataType::Time64(TimeUnit::Nanosecond) => {
    2473
    0
                    build_array_primitive!(Time64NanosecondArray, Time64Nanosecond)
    2474
                }
    2475
    0
                DataType::Timestamp(TimeUnit::Second, tz) => {
    2476
    0
                    build_array_primitive_tz!(TimestampSecondArray, TimestampSecond, tz)
    2477
                }
    2478
    0
                DataType::Timestamp(TimeUnit::Millisecond, tz) => {
    2479
    0
                    build_array_primitive_tz!(
    2480
                        TimestampMillisecondArray,
    2481
                        TimestampMillisecond,
    2482
    0
                        tz
    2483
                    )
    2484
                }
    2485
    2
                DataType::Timestamp(TimeUnit::Microsecond, tz) => {
    2486
    2
                    build_array_primitive_tz!(
    2487
                        TimestampMicrosecondArray,
    2488
                        TimestampMicrosecond,
    2489
    2
                        tz
    2490
                    )
    2491
                }
    2492
    0
                DataType::Timestamp(TimeUnit::Nanosecond, tz) => {
    2493
    0
                    build_array_primitive_tz!(
    2494
                        TimestampNanosecondArray,
    2495
                        TimestampNanosecond,
    2496
    0
                        tz
    2497
                    )
    2498
                }
    2499
                DataType::Duration(TimeUnit::Second) => {
    2500
    0
                    build_array_primitive!(DurationSecondArray, DurationSecond)
    2501
                }
    2502
                DataType::Duration(TimeUnit::Millisecond) => {
    2503
    0
                    build_array_primitive!(DurationMillisecondArray, DurationMillisecond)
    2504
                }
    2505
                DataType::Duration(TimeUnit::Microsecond) => {
    2506
    0
                    build_array_primitive!(DurationMicrosecondArray, DurationMicrosecond)
    2507
                }
    2508
                DataType::Duration(TimeUnit::Nanosecond) => {
    2509
    0
                    build_array_primitive!(DurationNanosecondArray, DurationNanosecond)
    2510
                }
    2511
                DataType::Interval(IntervalUnit::DayTime) => {
    2512
    0
                    build_array_primitive!(IntervalDayTimeArray, IntervalDayTime)
    2513
                }
    2514
                DataType::Interval(IntervalUnit::YearMonth) => {
    2515
    0
                    build_array_primitive!(IntervalYearMonthArray, IntervalYearMonth)
    2516
                }
    2517
                DataType::Interval(IntervalUnit::MonthDayNano) => {
    2518
    0
                    build_array_primitive!(IntervalMonthDayNanoArray, IntervalMonthDayNano)
    2519
                }
    2520
                DataType::FixedSizeList(_, _) => {
    2521
                    // arrow::compute::concat does not allow inconsistent types including the size of FixedSizeList.
    2522
                    // The length of nulls here we got is 1, so we need to resize the length of nulls to
    2523
                    // the length of non-nulls.
    2524
    0
                    let mut arrays =
    2525
    0
                        scalars.map(|s| s.to_array()).collect::<Result<Vec<_>>>()?;
    2526
    0
                    let first_non_null_data_type = arrays
    2527
    0
                        .iter()
    2528
    0
                        .find(|sv| !sv.is_null(0))
    2529
    0
                        .map(|sv| sv.data_type().to_owned());
    2530
    0
                    if let Some(DataType::FixedSizeList(f, l)) = first_non_null_data_type {
    2531
    0
                        for array in arrays.iter_mut() {
    2532
    0
                            if array.is_null(0) {
    2533
    0
                                *array = Arc::new(FixedSizeListArray::new_null(
    2534
    0
                                    Arc::clone(&f),
    2535
    0
                                    l,
    2536
    0
                                    1,
    2537
    0
                                ));
    2538
    0
                            }
    2539
                        }
    2540
    0
                    }
    2541
    0
                    let arrays = arrays.iter().map(|a| a.as_ref()).collect::<Vec<_>>();
    2542
    0
                    arrow::compute::concat(arrays.as_slice())?
    2543
                }
    2544
                DataType::List(_)
    2545
                | DataType::LargeList(_)
    2546
                | DataType::Map(_, _)
    2547
                | DataType::Struct(_)
    2548
                | DataType::Union(_, _) => {
    2549
    23
                    let arrays19 = scalars19.map19(|s| s.to_array()).collect19::<Result<Vec<_>>>()?0;
    2550
    23
                    let arrays19 = arrays.iter()19.map19(|a| a.as_ref()).collect19::<Vec<_>>();
    2551
    19
                    arrow::compute::concat(arrays.as_slice())?0
    2552
                }
    2553
    0
                DataType::Dictionary(key_type, value_type) => {
    2554
                    // create the values array
    2555
    0
                    let value_scalars = scalars
    2556
    0
                        .map(|scalar| match scalar {
    2557
    0
                            ScalarValue::Dictionary(inner_key_type, scalar) => {
    2558
    0
                                if &inner_key_type == key_type {
    2559
    0
                                    Ok(*scalar)
    2560
                                } else {
    2561
    0
                                    _exec_err!("Expected inner key type of {key_type} but found: {inner_key_type}, value was ({scalar:?})")
    2562
                                }
    2563
                            }
    2564
                            _ => {
    2565
    0
                                _exec_err!(
    2566
    0
                                    "Expected scalar of type {value_type} but found: {scalar} {scalar:?}"
    2567
                                )
    2568
                            }
    2569
    0
                        })
    2570
    0
                        .collect::<Result<Vec<_>>>()?;
    2571
    2572
    0
                    let values = Self::iter_to_array(value_scalars)?;
    2573
    0
                    assert_eq!(values.data_type(), value_type.as_ref());
    2574
    2575
    0
                    match key_type.as_ref() {
    2576
    0
                        DataType::Int8 => dict_from_values::<Int8Type>(values)?,
    2577
    0
                        DataType::Int16 => dict_from_values::<Int16Type>(values)?,
    2578
    0
                        DataType::Int32 => dict_from_values::<Int32Type>(values)?,
    2579
    0
                        DataType::Int64 => dict_from_values::<Int64Type>(values)?,
    2580
    0
                        DataType::UInt8 => dict_from_values::<UInt8Type>(values)?,
    2581
    0
                        DataType::UInt16 => dict_from_values::<UInt16Type>(values)?,
    2582
    0
                        DataType::UInt32 => dict_from_values::<UInt32Type>(values)?,
    2583
    0
                        DataType::UInt64 => dict_from_values::<UInt64Type>(values)?,
    2584
    0
                        _ => unreachable!("Invalid dictionary keys type: {}", key_type),
    2585
                    }
    2586
                }
    2587
    0
                DataType::FixedSizeBinary(size) => {
    2588
    0
                    let array = scalars
    2589
    0
                        .map(|sv| {
    2590
    0
                            if let ScalarValue::FixedSizeBinary(_, v) = sv {
    2591
    0
                                Ok(v)
    2592
                            } else {
    2593
    0
                                _exec_err!(
    2594
    0
                                    "Inconsistent types in ScalarValue::iter_to_array. \
    2595
    0
                                    Expected {data_type}, got {sv:?}"
    2596
                                )
    2597
                            }
    2598
    0
                        })
    2599
    0
                        .collect::<Result<Vec<_>>>()?;
    2600
    0
                    let array = FixedSizeBinaryArray::try_from_sparse_iter_with_size(
    2601
    0
                        array.into_iter(),
    2602
    0
                        *size,
    2603
    0
                    )?;
    2604
    0
                    Arc::new(array)
    2605
                }
    2606
                // explicitly enumerate unsupported types so newly added
    2607
                // types must be acknowledged, Time32 and Time64 types are
    2608
                // not supported if the TimeUnit is not valid (Time32 can
    2609
                // only be used with Second and Millisecond, Time64 only
    2610
                // with Microsecond and Nanosecond)
    2611
                DataType::Time32(TimeUnit::Microsecond)
    2612
                | DataType::Time32(TimeUnit::Nanosecond)
    2613
                | DataType::Time64(TimeUnit::Second)
    2614
                | DataType::Time64(TimeUnit::Millisecond)
    2615
                | DataType::RunEndEncoded(_, _)
    2616
                | DataType::ListView(_)
    2617
                | DataType::LargeListView(_) => {
    2618
    0
                    return _not_impl_err!(
    2619
    0
                        "Unsupported creation of {:?} array from ScalarValue {:?}",
    2620
                        data_type,
    2621
    0
                        scalars.peek()
    2622
                    );
    2623
                }
    2624
            };
    2625
    159
            Ok(array)
    2626
    159
        }
    2627
    2628
    0
        fn iter_to_null_array(
    2629
    0
            scalars: impl IntoIterator<Item = ScalarValue>,
    2630
    0
        ) -> Result<ArrayRef> {
    2631
    0
            let length = scalars.into_iter().try_fold(
    2632
                0usize,
    2633
    0
                |r, element: ScalarValue| match element {
    2634
    0
                    ScalarValue::Null => Ok::<usize, DataFusionError>(r + 1),
    2635
    0
                    s => {
    2636
    0
                        _internal_err!("Expected ScalarValue::Null element. Received {s:?}")
    2637
                    }
    2638
    0
                },
    2639
    0
            )?;
    2640
    0
            Ok(new_null_array(&DataType::Null, length))
    2641
    0
        }
    2642
    2643
    0
        fn iter_to_decimal32_array(
    2644
    0
            scalars: impl IntoIterator<Item = ScalarValue>,
    2645
    0
            precision: u8,
    2646
    0
            scale: i8,
    2647
    0
        ) -> Result<Decimal32Array> {
    2648
    0
            let array = scalars
    2649
    0
                .into_iter()
    2650
    0
                .map(|element: ScalarValue| match element {
    2651
    0
                    ScalarValue::Decimal32(v1, _, _) => Ok(v1),
    2652
    0
                    s => {
    2653
    0
                        _internal_err!("Expected ScalarValue::Null element. Received {s:?}")
    2654
                    }
    2655
    0
                })
    2656
    0
                .collect::<Result<Decimal32Array>>()?
    2657
    0
                .with_precision_and_scale(precision, scale)?;
    2658
    0
            Ok(array)
    2659
    0
        }
    2660
    2661
    0
        fn iter_to_decimal64_array(
    2662
    0
            scalars: impl IntoIterator<Item = ScalarValue>,
    2663
    0
            precision: u8,
    2664
    0
            scale: i8,
    2665
    0
        ) -> Result<Decimal64Array> {
    2666
    0
            let array = scalars
    2667
    0
                .into_iter()
    2668
    0
                .map(|element: ScalarValue| match element {
    2669
    0
                    ScalarValue::Decimal64(v1, _, _) => Ok(v1),
    2670
    0
                    s => {
    2671
    0
                        _internal_err!("Expected ScalarValue::Null element. Received {s:?}")
    2672
                    }
    2673
    0
                })
    2674
    0
                .collect::<Result<Decimal64Array>>()?
    2675
    0
                .with_precision_and_scale(precision, scale)?;
    2676
    0
            Ok(array)
    2677
    0
        }
    2678
    2679
    8
        fn iter_to_decimal128_array(
    2680
    8
            scalars: impl IntoIterator<Item = ScalarValue>,
    2681
    8
            precision: u8,
    2682
    8
            scale: i8,
    2683
    8
        ) -> Result<Decimal128Array> {
    2684
    8
            let array = scalars
    2685
    8
                .into_iter()
    2686
    414
                .map8(|element: ScalarValue| match element {
    2687
    414
                    ScalarValue::Decimal128(v1, _, _) => Ok(v1),
    2688
    0
                    s => {
    2689
    0
                        _internal_err!("Expected ScalarValue::Null element. Received {s:?}")
    2690
                    }
    2691
    414
                })
    2692
    8
                .collect::<Result<Decimal128Array>>()?0
    2693
    8
                .with_precision_and_scale(precision, scale)?0;
    2694
    8
            Ok(array)
    2695
    8
        }
    2696
    2697
    0
        fn iter_to_decimal256_array(
    2698
    0
            scalars: impl IntoIterator<Item = ScalarValue>,
    2699
    0
            precision: u8,
    2700
    0
            scale: i8,
    2701
    0
        ) -> Result<Decimal256Array> {
    2702
    0
            let array = scalars
    2703
    0
                .into_iter()
    2704
    0
                .map(|element: ScalarValue| match element {
    2705
    0
                    ScalarValue::Decimal256(v1, _, _) => Ok(v1),
    2706
    0
                    s => {
    2707
    0
                        _internal_err!(
    2708
    0
                            "Expected ScalarValue::Decimal256 element. Received {s:?}"
    2709
                        )
    2710
                    }
    2711
    0
                })
    2712
    0
                .collect::<Result<Decimal256Array>>()?
    2713
    0
                .with_precision_and_scale(precision, scale)?;
    2714
    0
            Ok(array)
    2715
    0
        }
    2716
    2717
    0
        fn build_decimal32_array(
    2718
    0
            value: Option<i32>,
    2719
    0
            precision: u8,
    2720
    0
            scale: i8,
    2721
    0
            size: usize,
    2722
    0
        ) -> Result<Decimal32Array> {
    2723
    0
            Ok(match value {
    2724
    0
                Some(val) => Decimal32Array::from(vec![val; size])
    2725
    0
                    .with_precision_and_scale(precision, scale)?,
    2726
                None => {
    2727
    0
                    let mut builder = Decimal32Array::builder(size)
    2728
    0
                        .with_precision_and_scale(precision, scale)?;
    2729
    0
                    builder.append_nulls(size);
    2730
    0
                    builder.finish()
    2731
                }
    2732
            })
    2733
    0
        }
    2734
    2735
    0
        fn build_decimal64_array(
    2736
    0
            value: Option<i64>,
    2737
    0
            precision: u8,
    2738
    0
            scale: i8,
    2739
    0
            size: usize,
    2740
    0
        ) -> Result<Decimal64Array> {
    2741
    0
            Ok(match value {
    2742
    0
                Some(val) => Decimal64Array::from(vec![val; size])
    2743
    0
                    .with_precision_and_scale(precision, scale)?,
    2744
                None => {
    2745
    0
                    let mut builder = Decimal64Array::builder(size)
    2746
    0
                        .with_precision_and_scale(precision, scale)?;
    2747
    0
                    builder.append_nulls(size);
    2748
    0
                    builder.finish()
    2749
                }
    2750
            })
    2751
    0
        }
    2752
    2753
    0
        fn build_decimal128_array(
    2754
    0
            value: Option<i128>,
    2755
    0
            precision: u8,
    2756
    0
            scale: i8,
    2757
    0
            size: usize,
    2758
    0
        ) -> Result<Decimal128Array> {
    2759
    0
            Ok(match value {
    2760
    0
                Some(val) => Decimal128Array::from(vec![val; size])
    2761
    0
                    .with_precision_and_scale(precision, scale)?,
    2762
                None => {
    2763
    0
                    let mut builder = Decimal128Array::builder(size)
    2764
    0
                        .with_precision_and_scale(precision, scale)?;
    2765
    0
                    builder.append_nulls(size);
    2766
    0
                    builder.finish()
    2767
                }
    2768
            })
    2769
    0
        }
    2770
    2771
    0
        fn build_decimal256_array(
    2772
    0
            value: Option<i256>,
    2773
    0
            precision: u8,
    2774
    0
            scale: i8,
    2775
    0
            size: usize,
    2776
    0
        ) -> Result<Decimal256Array> {
    2777
    0
            Ok(repeat_n(value, size)
    2778
    0
                .collect::<Decimal256Array>()
    2779
    0
                .with_precision_and_scale(precision, scale)?)
    2780
    0
        }
    2781
    2782
        /// Converts `Vec<ScalarValue>` where each element has type corresponding to
    2783
        /// `data_type`, to a single element [`ListArray`].
    2784
        ///
    2785
        /// Example
    2786
        /// ```
    2787
        /// use arrow::array::{Int32Array, ListArray};
    2788
        /// use arrow::datatypes::{DataType, Int32Type};
    2789
        /// use datafusion_common::cast::as_list_array;
    2790
        /// use datafusion_common::ScalarValue;
    2791
        ///
    2792
        /// let scalars = vec![
    2793
        ///     ScalarValue::Int32(Some(1)),
    2794
        ///     ScalarValue::Int32(None),
    2795
        ///     ScalarValue::Int32(Some(2)),
    2796
        /// ];
    2797
        ///
    2798
        /// let result = ScalarValue::new_list(&scalars, &DataType::Int32, true);
    2799
        ///
    2800
        /// let expected = ListArray::from_iter_primitive::<Int32Type, _, _>(vec![Some(vec![
    2801
        ///     Some(1),
    2802
        ///     None,
    2803
        ///     Some(2),
    2804
        /// ])]);
    2805
        ///
    2806
        /// assert_eq!(*result, expected);
    2807
        /// ```
    2808
    0
        pub fn new_list(
    2809
    0
            values: &[ScalarValue],
    2810
    0
            data_type: &DataType,
    2811
    0
            nullable: bool,
    2812
    0
        ) -> Arc<ListArray> {
    2813
    0
            let values = if values.is_empty() {
    2814
    0
                new_empty_array(data_type)
    2815
            } else {
    2816
    0
                Self::iter_to_array(values.iter().cloned()).unwrap()
    2817
            };
    2818
    0
            Arc::new(
    2819
    0
                SingleRowListArrayBuilder::new(values)
    2820
    0
                    .with_nullable(nullable)
    2821
    0
                    .build_list_array(),
    2822
            )
    2823
    0
        }
    2824
    2825
        /// Same as [`ScalarValue::new_list`] but with nullable set to true.
    2826
    0
        pub fn new_list_nullable(
    2827
    0
            values: &[ScalarValue],
    2828
    0
            data_type: &DataType,
    2829
    0
        ) -> Arc<ListArray> {
    2830
    0
            Self::new_list(values, data_type, true)
    2831
    0
        }
    2832
    2833
        /// Create ListArray with Null with specific data type
    2834
        ///
    2835
        /// - new_null_list(i32, nullable, 1): `ListArray[NULL]`
    2836
    0
        pub fn new_null_list(data_type: DataType, nullable: bool, null_len: usize) -> Self {
    2837
    0
            let data_type = DataType::List(Field::new_list_field(data_type, nullable).into());
    2838
    0
            Self::List(Arc::new(ListArray::from(ArrayData::new_null(
    2839
    0
                &data_type, null_len,
    2840
    0
            ))))
    2841
    0
        }
    2842
    2843
        /// Converts `IntoIterator<Item = ScalarValue>` where each element has type corresponding to
    2844
        /// `data_type`, to a [`ListArray`].
    2845
        ///
    2846
        /// Example
    2847
        /// ```
    2848
        /// use arrow::array::{Int32Array, ListArray};
    2849
        /// use arrow::datatypes::{DataType, Int32Type};
    2850
        /// use datafusion_common::cast::as_list_array;
    2851
        /// use datafusion_common::ScalarValue;
    2852
        ///
    2853
        /// let scalars = vec![
    2854
        ///     ScalarValue::Int32(Some(1)),
    2855
        ///     ScalarValue::Int32(None),
    2856
        ///     ScalarValue::Int32(Some(2)),
    2857
        /// ];
    2858
        ///
    2859
        /// let result =
    2860
        ///     ScalarValue::new_list_from_iter(scalars.into_iter(), &DataType::Int32, true);
    2861
        ///
    2862
        /// let expected = ListArray::from_iter_primitive::<Int32Type, _, _>(vec![Some(vec![
    2863
        ///     Some(1),
    2864
        ///     None,
    2865
        ///     Some(2),
    2866
        /// ])]);
    2867
        ///
    2868
        /// assert_eq!(*result, expected);
    2869
        /// ```
    2870
    0
        pub fn new_list_from_iter(
    2871
    0
            values: impl IntoIterator<Item = ScalarValue> + ExactSizeIterator,
    2872
    0
            data_type: &DataType,
    2873
    0
            nullable: bool,
    2874
    0
        ) -> Arc<ListArray> {
    2875
    0
            let values = if values.len() == 0 {
    2876
    0
                new_empty_array(data_type)
    2877
            } else {
    2878
    0
                Self::iter_to_array(values).unwrap()
    2879
            };
    2880
    0
            Arc::new(
    2881
    0
                SingleRowListArrayBuilder::new(values)
    2882
    0
                    .with_nullable(nullable)
    2883
    0
                    .build_list_array(),
    2884
            )
    2885
    0
        }
    2886
    2887
        /// Converts `Vec<ScalarValue>` where each element has type corresponding to
    2888
        /// `data_type`, to a [`LargeListArray`].
    2889
        ///
    2890
        /// Example
    2891
        /// ```
    2892
        /// use arrow::array::{Int32Array, LargeListArray};
    2893
        /// use arrow::datatypes::{DataType, Int32Type};
    2894
        /// use datafusion_common::cast::as_large_list_array;
    2895
        /// use datafusion_common::ScalarValue;
    2896
        ///
    2897
        /// let scalars = vec![
    2898
        ///     ScalarValue::Int32(Some(1)),
    2899
        ///     ScalarValue::Int32(None),
    2900
        ///     ScalarValue::Int32(Some(2)),
    2901
        /// ];
    2902
        ///
    2903
        /// let result = ScalarValue::new_large_list(&scalars, &DataType::Int32);
    2904
        ///
    2905
        /// let expected =
    2906
        ///     LargeListArray::from_iter_primitive::<Int32Type, _, _>(vec![Some(vec![
    2907
        ///         Some(1),
    2908
        ///         None,
    2909
        ///         Some(2),
    2910
        ///     ])]);
    2911
        ///
    2912
        /// assert_eq!(*result, expected);
    2913
        /// ```
    2914
    0
        pub fn new_large_list(
    2915
    0
            values: &[ScalarValue],
    2916
    0
            data_type: &DataType,
    2917
    0
        ) -> Arc<LargeListArray> {
    2918
    0
            let values = if values.is_empty() {
    2919
    0
                new_empty_array(data_type)
    2920
            } else {
    2921
    0
                Self::iter_to_array(values.iter().cloned()).unwrap()
    2922
            };
    2923
    0
            Arc::new(SingleRowListArrayBuilder::new(values).build_large_list_array())
    2924
    0
        }
    2925
    2926
        /// Converts a scalar value into an array of `size` rows.
    2927
        ///
    2928
        /// # Errors
    2929
        ///
    2930
        /// Errors if `self` is
    2931
        /// - a decimal that fails be converted to a decimal array of size
    2932
        /// - a `FixedsizeList` that fails to be concatenated into an array of size
    2933
        /// - a `List` that fails to be concatenated into an array of size
    2934
        /// - a `Dictionary` that fails be converted to a dictionary array of size
    2935
    540
        pub fn to_array_of_size(&self, size: usize) -> Result<ArrayRef> {
    2936
    540
            Ok(match self {
    2937
    0
                ScalarValue::Decimal32(e, precision, scale) => Arc::new(
    2938
    0
                    ScalarValue::build_decimal32_array(*e, *precision, *scale, size)?,
    2939
                ),
    2940
    0
                ScalarValue::Decimal64(e, precision, scale) => Arc::new(
    2941
    0
                    ScalarValue::build_decimal64_array(*e, *precision, *scale, size)?,
    2942
                ),
    2943
    0
                ScalarValue::Decimal128(e, precision, scale) => Arc::new(
    2944
    0
                    ScalarValue::build_decimal128_array(*e, *precision, *scale, size)?,
    2945
                ),
    2946
    0
                ScalarValue::Decimal256(e, precision, scale) => Arc::new(
    2947
    0
                    ScalarValue::build_decimal256_array(*e, *precision, *scale, size)?,
    2948
                ),
    2949
    0
                ScalarValue::Boolean(e) => match e {
    2950
    0
                    None => new_null_array(&DataType::Boolean, size),
    2951
                    Some(true) => {
    2952
    0
                        Arc::new(BooleanArray::new(BooleanBuffer::new_set(size), None))
    2953
    0
                            as ArrayRef
    2954
                    }
    2955
                    Some(false) => {
    2956
    0
                        Arc::new(BooleanArray::new(BooleanBuffer::new_unset(size), None))
    2957
    0
                            as ArrayRef
    2958
                    }
    2959
                },
    2960
    0
                ScalarValue::Float64(e) => {
    2961
    0
                    build_array_from_option!(Float64, Float64Array, e, size)
    2962
                }
    2963
    5
                ScalarValue::Float32(e) => {
    2964
    5
                    build_array_from_option!(Float32, Float32Array, e, size0)
    2965
                }
    2966
    0
                ScalarValue::Float16(e) => {
    2967
    0
                    build_array_from_option!(Float16, Float16Array, e, size)
    2968
                }
    2969
    0
                ScalarValue::Int8(e) => build_array_from_option!(Int8, Int8Array, e, size),
    2970
    0
                ScalarValue::Int16(e) => build_array_from_option!(Int16, Int16Array, e, size),
    2971
    431
                ScalarValue::Int32(e) => build_array_from_option!(Int32, Int32Array, e, size0),
    2972
    2
                ScalarValue::Int64(e) => build_array_from_option!(Int64, Int64Array, e, size0),
    2973
    0
                ScalarValue::UInt8(e) => build_array_from_option!(UInt8, UInt8Array, e, size),
    2974
    0
                ScalarValue::UInt16(e) => {
    2975
    0
                    build_array_from_option!(UInt16, UInt16Array, e, size)
    2976
                }
    2977
    0
                ScalarValue::UInt32(e) => {
    2978
    0
                    build_array_from_option!(UInt32, UInt32Array, e, size)
    2979
                }
    2980
    0
                ScalarValue::UInt64(e) => {
    2981
    0
                    build_array_from_option!(UInt64, UInt64Array, e, size)
    2982
                }
    2983
    0
                ScalarValue::TimestampSecond(e, tz_opt) => {
    2984
    0
                    build_timestamp_array_from_option!(
    2985
    0
                        TimeUnit::Second,
    2986
    0
                        tz_opt.clone(),
    2987
                        TimestampSecondArray,
    2988
    0
                        e,
    2989
    0
                        size
    2990
                    )
    2991
                }
    2992
    0
                ScalarValue::TimestampMillisecond(e, tz_opt) => {
    2993
    0
                    build_timestamp_array_from_option!(
    2994
    0
                        TimeUnit::Millisecond,
    2995
    0
                        tz_opt.clone(),
    2996
                        TimestampMillisecondArray,
    2997
    0
                        e,
    2998
    0
                        size
    2999
                    )
    3000
                }
    3001
    3002
    0
                ScalarValue::TimestampMicrosecond(e, tz_opt) => {
    3003
    0
                    build_timestamp_array_from_option!(
    3004
    0
                        TimeUnit::Microsecond,
    3005
    0
                        tz_opt.clone(),
    3006
                        TimestampMicrosecondArray,
    3007
    0
                        e,
    3008
    0
                        size
    3009
                    )
    3010
                }
    3011
    0
                ScalarValue::TimestampNanosecond(e, tz_opt) => {
    3012
    0
                    build_timestamp_array_from_option!(
    3013
    0
                        TimeUnit::Nanosecond,
    3014
    0
                        tz_opt.clone(),
    3015
                        TimestampNanosecondArray,
    3016
    0
                        e,
    3017
    0
                        size
    3018
                    )
    3019
                }
    3020
    20
                ScalarValue::Utf8(e) => match e {
    3021
    20
                    Some(value) => {
    3022
    20
                        Arc::new(StringArray::from_iter_values(repeat_n(value, size)))
    3023
                    }
    3024
    0
                    None => new_null_array(&DataType::Utf8, size),
    3025
                },
    3026
    0
                ScalarValue::Utf8View(e) => match e {
    3027
    0
                    Some(value) => {
    3028
    0
                        Arc::new(StringViewArray::from_iter_values(repeat_n(value, size)))
    3029
                    }
    3030
    0
                    None => new_null_array(&DataType::Utf8View, size),
    3031
                },
    3032
    0
                ScalarValue::LargeUtf8(e) => match e {
    3033
    0
                    Some(value) => {
    3034
    0
                        Arc::new(LargeStringArray::from_iter_values(repeat_n(value, size)))
    3035
                    }
    3036
    0
                    None => new_null_array(&DataType::LargeUtf8, size),
    3037
                },
    3038
    16
                ScalarValue::Binary(e) => match e {
    3039
    16
                    Some(value) => Arc::new(
    3040
    16
                        repeat_n(Some(value.as_slice()), size).collect::<BinaryArray>(),
    3041
    16
                    ),
    3042
    0
                    None => new_null_array(&DataType::Binary, size),
    3043
                },
    3044
    0
                ScalarValue::BinaryView(e) => match e {
    3045
    0
                    Some(value) => Arc::new(
    3046
    0
                        repeat_n(Some(value.as_slice()), size).collect::<BinaryViewArray>(),
    3047
    0
                    ),
    3048
    0
                    None => new_null_array(&DataType::BinaryView, size),
    3049
                },
    3050
    0
                ScalarValue::FixedSizeBinary(s, e) => match e {
    3051
    0
                    Some(value) => Arc::new(
    3052
    0
                        FixedSizeBinaryArray::try_from_sparse_iter_with_size(
    3053
    0
                            repeat_n(Some(value.as_slice()), size),
    3054
    0
                            *s,
    3055
    0
                        )
    3056
    0
                        .unwrap(),
    3057
    0
                    ),
    3058
    0
                    None => Arc::new(FixedSizeBinaryArray::new_null(*s, size)),
    3059
                },
    3060
    0
                ScalarValue::LargeBinary(e) => match e {
    3061
    0
                    Some(value) => Arc::new(
    3062
    0
                        repeat_n(Some(value.as_slice()), size).collect::<LargeBinaryArray>(),
    3063
    0
                    ),
    3064
    0
                    None => new_null_array(&DataType::LargeBinary, size),
    3065
                },
    3066
    0
                ScalarValue::List(arr) => {
    3067
    0
                    if size == 1 {
    3068
    0
                        return Ok(Arc::clone(arr) as Arc<dyn Array>);
    3069
    0
                    }
    3070
    0
                    Self::list_to_array_of_size(arr.as_ref() as &dyn Array, size)?
    3071
                }
    3072
    0
                ScalarValue::LargeList(arr) => {
    3073
    0
                    if size == 1 {
    3074
    0
                        return Ok(Arc::clone(arr) as Arc<dyn Array>);
    3075
    0
                    }
    3076
    0
                    Self::list_to_array_of_size(arr.as_ref() as &dyn Array, size)?
    3077
                }
    3078
    0
                ScalarValue::FixedSizeList(arr) => {
    3079
    0
                    if size == 1 {
    3080
    0
                        return Ok(Arc::clone(arr) as Arc<dyn Array>);
    3081
    0
                    }
    3082
    0
                    Self::list_to_array_of_size(arr.as_ref() as &dyn Array, size)?
    3083
                }
    3084
    38
                ScalarValue::Struct(arr) => {
    3085
    38
                    if size == 1 {
    3086
    38
                        return Ok(Arc::clone(arr) as Arc<dyn Array>);
    3087
    0
                    }
    3088
    0
                    Self::list_to_array_of_size(arr.as_ref() as &dyn Array, size)?
    3089
                }
    3090
    0
                ScalarValue::Map(arr) => {
    3091
    0
                    if size == 1 {
    3092
    0
                        return Ok(Arc::clone(arr) as Arc<dyn Array>);
    3093
    0
                    }
    3094
    0
                    Self::list_to_array_of_size(arr.as_ref() as &dyn Array, size)?
    3095
                }
    3096
    0
                ScalarValue::Date32(e) => {
    3097
    0
                    build_array_from_option!(Date32, Date32Array, e, size)
    3098
                }
    3099
    0
                ScalarValue::Date64(e) => {
    3100
    0
                    build_array_from_option!(Date64, Date64Array, e, size)
    3101
                }
    3102
    0
                ScalarValue::Time32Second(e) => {
    3103
    0
                    build_array_from_option!(
    3104
                        Time32,
    3105
                        TimeUnit::Second,
    3106
                        Time32SecondArray,
    3107
    0
                        e,
    3108
    0
                        size
    3109
                    )
    3110
                }
    3111
    0
                ScalarValue::Time32Millisecond(e) => {
    3112
    0
                    build_array_from_option!(
    3113
                        Time32,
    3114
                        TimeUnit::Millisecond,
    3115
                        Time32MillisecondArray,
    3116
    0
                        e,
    3117
    0
                        size
    3118
                    )
    3119
                }
    3120
    0
                ScalarValue::Time64Microsecond(e) => {
    3121
    0
                    build_array_from_option!(
    3122
                        Time64,
    3123
                        TimeUnit::Microsecond,
    3124
                        Time64MicrosecondArray,
    3125
    0
                        e,
    3126
    0
                        size
    3127
                    )
    3128
                }
    3129
    0
                ScalarValue::Time64Nanosecond(e) => {
    3130
    0
                    build_array_from_option!(
    3131
                        Time64,
    3132
                        TimeUnit::Nanosecond,
    3133
                        Time64NanosecondArray,
    3134
    0
                        e,
    3135
    0
                        size
    3136
                    )
    3137
                }
    3138
    0
                ScalarValue::IntervalDayTime(e) => build_array_from_option!(
    3139
                    Interval,
    3140
                    IntervalUnit::DayTime,
    3141
                    IntervalDayTimeArray,
    3142
    0
                    e,
    3143
    0
                    size
    3144
                ),
    3145
    0
                ScalarValue::IntervalYearMonth(e) => build_array_from_option!(
    3146
                    Interval,
    3147
                    IntervalUnit::YearMonth,
    3148
                    IntervalYearMonthArray,
    3149
    0
                    e,
    3150
    0
                    size
    3151
                ),
    3152
    0
                ScalarValue::IntervalMonthDayNano(e) => build_array_from_option!(
    3153
                    Interval,
    3154
                    IntervalUnit::MonthDayNano,
    3155
                    IntervalMonthDayNanoArray,
    3156
    0
                    e,
    3157
    0
                    size
    3158
                ),
    3159
    0
                ScalarValue::DurationSecond(e) => build_array_from_option!(
    3160
                    Duration,
    3161
                    TimeUnit::Second,
    3162
                    DurationSecondArray,
    3163
    0
                    e,
    3164
    0
                    size
    3165
                ),
    3166
    0
                ScalarValue::DurationMillisecond(e) => build_array_from_option!(
    3167
                    Duration,
    3168
                    TimeUnit::Millisecond,
    3169
                    DurationMillisecondArray,
    3170
    0
                    e,
    3171
    0
                    size
    3172
                ),
    3173
    0
                ScalarValue::DurationMicrosecond(e) => build_array_from_option!(
    3174
                    Duration,
    3175
                    TimeUnit::Microsecond,
    3176
                    DurationMicrosecondArray,
    3177
    0
                    e,
    3178
    0
                    size
    3179
                ),
    3180
    0
                ScalarValue::DurationNanosecond(e) => build_array_from_option!(
    3181
                    Duration,
    3182
                    TimeUnit::Nanosecond,
    3183
                    DurationNanosecondArray,
    3184
    0
                    e,
    3185
    0
                    size
    3186
                ),
    3187
    0
                ScalarValue::Union(value, fields, mode) => match value {
    3188
    0
                    Some((v_id, value)) => {
    3189
    0
                        let mut new_fields = Vec::with_capacity(fields.len());
    3190
    0
                        let mut child_arrays = Vec::<ArrayRef>::with_capacity(fields.len());
    3191
    0
                        for (f_id, field) in fields.iter() {
    3192
    0
                            let ar = if f_id == *v_id {
    3193
    0
                                value.to_array_of_size(size)?
    3194
                            } else {
    3195
    0
                                let dt = field.data_type();
    3196
    0
                                match mode {
    3197
    0
                                    UnionMode::Sparse => new_null_array(dt, size),
    3198
                                    // In a dense union, only the child with values needs to be
    3199
                                    // allocated
    3200
    0
                                    UnionMode::Dense => new_null_array(dt, 0),
    3201
                                }
    3202
                            };
    3203
    0
                            let field = (**field).clone();
    3204
    0
                            child_arrays.push(ar);
    3205
    0
                            new_fields.push(field.clone());
    3206
                        }
    3207
    0
                        let type_ids = repeat_n(*v_id, size);
    3208
    0
                        let type_ids = ScalarBuffer::<i8>::from_iter(type_ids);
    3209
    0
                        let value_offsets = match mode {
    3210
    0
                            UnionMode::Sparse => None,
    3211
    0
                            UnionMode::Dense => Some(ScalarBuffer::from_iter(0..size as i32)),
    3212
                        };
    3213
    0
                        let ar = UnionArray::try_new(
    3214
    0
                            fields.clone(),
    3215
    0
                            type_ids,
    3216
    0
                            value_offsets,
    3217
    0
                            child_arrays,
    3218
                        )
    3219
    0
                        .map_err(|e| DataFusionError::ArrowError(Box::new(e), None))?;
    3220
    0
                        Arc::new(ar)
    3221
                    }
    3222
                    None => {
    3223
    0
                        let dt = self.data_type();
    3224
    0
                        new_null_array(&dt, size)
    3225
                    }
    3226
                },
    3227
    0
                ScalarValue::Dictionary(key_type, v) => {
    3228
                    // values array is one element long (the value)
    3229
    0
                    match key_type.as_ref() {
    3230
    0
                        DataType::Int8 => dict_from_scalar::<Int8Type>(v, size)?,
    3231
    0
                        DataType::Int16 => dict_from_scalar::<Int16Type>(v, size)?,
    3232
    0
                        DataType::Int32 => dict_from_scalar::<Int32Type>(v, size)?,
    3233
    0
                        DataType::Int64 => dict_from_scalar::<Int64Type>(v, size)?,
    3234
    0
                        DataType::UInt8 => dict_from_scalar::<UInt8Type>(v, size)?,
    3235
    0
                        DataType::UInt16 => dict_from_scalar::<UInt16Type>(v, size)?,
    3236
    0
                        DataType::UInt32 => dict_from_scalar::<UInt32Type>(v, size)?,
    3237
    0
                        DataType::UInt64 => dict_from_scalar::<UInt64Type>(v, size)?,
    3238
    0
                        _ => unreachable!("Invalid dictionary keys type: {}", key_type),
    3239
                    }
    3240
                }
    3241
    28
                ScalarValue::Null => get_or_create_cached_null_array(size),
    3242
            })
    3243
    540
        }
    3244
    3245
    412
        fn get_decimal_value_from_array(
    3246
    412
            array: &dyn Array,
    3247
    412
            index: usize,
    3248
    412
            precision: u8,
    3249
    412
            scale: i8,
    3250
    412
        ) -> Result<ScalarValue> {
    3251
    412
            match array.data_type() {
    3252
                DataType::Decimal32(_, _) => {
    3253
    0
                    let array = as_decimal32_array(array)?;
    3254
    0
                    if array.is_null(index) {
    3255
    0
                        Ok(ScalarValue::Decimal32(None, precision, scale))
    3256
                    } else {
    3257
    0
                        let value = array.value(index);
    3258
    0
                        Ok(ScalarValue::Decimal32(Some(value), precision, scale))
    3259
                    }
    3260
                }
    3261
                DataType::Decimal64(_, _) => {
    3262
    0
                    let array = as_decimal64_array(array)?;
    3263
    0
                    if array.is_null(index) {
    3264
    0
                        Ok(ScalarValue::Decimal64(None, precision, scale))
    3265
                    } else {
    3266
    0
                        let value = array.value(index);
    3267
    0
                        Ok(ScalarValue::Decimal64(Some(value), precision, scale))
    3268
                    }
    3269
                }
    3270
                DataType::Decimal128(_, _) => {
    3271
    412
                    let array = as_decimal128_array(array)?0;
    3272
    412
                    if array.is_null(index) {
    3273
    0
                        Ok(ScalarValue::Decimal128(None, precision, scale))
    3274
                    } else {
    3275
    412
                        let value = array.value(index);
    3276
    412
                        Ok(ScalarValue::Decimal128(Some(value), precision, scale))
    3277
                    }
    3278
                }
    3279
                DataType::Decimal256(_, _) => {
    3280
    0
                    let array = as_decimal256_array(array)?;
    3281
    0
                    if array.is_null(index) {
    3282
    0
                        Ok(ScalarValue::Decimal256(None, precision, scale))
    3283
                    } else {
    3284
    0
                        let value = array.value(index);
    3285
    0
                        Ok(ScalarValue::Decimal256(Some(value), precision, scale))
    3286
                    }
    3287
                }
    3288
    0
                other => {
    3289
    0
                    unreachable!("Invalid type isn't decimal: {other:?}")
    3290
                }
    3291
            }
    3292
    412
        }
    3293
    3294
    0
        fn list_to_array_of_size(arr: &dyn Array, size: usize) -> Result<ArrayRef> {
    3295
    0
            let arrays = repeat_n(arr, size).collect::<Vec<_>>();
    3296
    0
            let ret = match !arrays.is_empty() {
    3297
    0
                true => arrow::compute::concat(arrays.as_slice())?,
    3298
    0
                false => arr.slice(0, 0),
    3299
            };
    3300
    0
            Ok(ret)
    3301
    0
        }
    3302
    3303
        /// Retrieve ScalarValue for each row in `array`
    3304
        ///
    3305
        /// Elements in `array` may be NULL, in which case the corresponding element in the returned vector is None.
    3306
        ///
    3307
        /// Example 1: Array (ScalarValue::Int32)
    3308
        /// ```
    3309
        /// use arrow::array::ListArray;
    3310
        /// use arrow::datatypes::{DataType, Int32Type};
    3311
        /// use datafusion_common::ScalarValue;
    3312
        ///
    3313
        /// // Equivalent to [[1,2,3], [4,5]]
    3314
        /// let list_arr = ListArray::from_iter_primitive::<Int32Type, _, _>(vec![
    3315
        ///     Some(vec![Some(1), Some(2), Some(3)]),
    3316
        ///     Some(vec![Some(4), Some(5)]),
    3317
        /// ]);
    3318
        ///
    3319
        /// // Convert the array into Scalar Values for each row
    3320
        /// let scalar_vec = ScalarValue::convert_array_to_scalar_vec(&list_arr).unwrap();
    3321
        ///
    3322
        /// let expected = vec![
    3323
        ///     Some(vec![
    3324
        ///         ScalarValue::Int32(Some(1)),
    3325
        ///         ScalarValue::Int32(Some(2)),
    3326
        ///         ScalarValue::Int32(Some(3)),
    3327
        ///     ]),
    3328
        ///     Some(vec![
    3329
        ///         ScalarValue::Int32(Some(4)),
    3330
        ///         ScalarValue::Int32(Some(5)),
    3331
        ///     ]),
    3332
        /// ];
    3333
        ///
    3334
        /// assert_eq!(scalar_vec, expected);
    3335
        /// ```
    3336
        ///
    3337
        /// Example 2: Nested array (ScalarValue::List)
    3338
        /// ```
    3339
        /// use arrow::array::ListArray;
    3340
        /// use arrow::datatypes::{DataType, Int32Type};
    3341
        /// use datafusion_common::utils::SingleRowListArrayBuilder;
    3342
        /// use datafusion_common::ScalarValue;
    3343
        /// use std::sync::Arc;
    3344
        ///
    3345
        /// let list_arr = ListArray::from_iter_primitive::<Int32Type, _, _>(vec![
    3346
        ///     Some(vec![Some(1), Some(2), Some(3)]),
    3347
        ///     Some(vec![Some(4), Some(5)]),
    3348
        /// ]);
    3349
        ///
    3350
        /// // Wrap into another layer of list, we got nested array as [ [[1,2,3], [4,5]] ]
    3351
        /// let list_arr = SingleRowListArrayBuilder::new(Arc::new(list_arr)).build_list_array();
    3352
        ///
    3353
        /// // Convert the array into Scalar Values for each row, we got 1D arrays in this example
    3354
        /// let scalar_vec = ScalarValue::convert_array_to_scalar_vec(&list_arr).unwrap();
    3355
        ///
    3356
        /// let l1 = ListArray::from_iter_primitive::<Int32Type, _, _>(vec![Some(vec![
    3357
        ///     Some(1),
    3358
        ///     Some(2),
    3359
        ///     Some(3),
    3360
        /// ])]);
    3361
        /// let l2 = ListArray::from_iter_primitive::<Int32Type, _, _>(vec![Some(vec![
    3362
        ///     Some(4),
    3363
        ///     Some(5),
    3364
        /// ])]);
    3365
        ///
    3366
        /// let expected = vec![Some(vec![
    3367
        ///     ScalarValue::List(Arc::new(l1)),
    3368
        ///     ScalarValue::List(Arc::new(l2)),
    3369
        /// ])];
    3370
        ///
    3371
        /// assert_eq!(scalar_vec, expected);
    3372
        /// ```
    3373
        ///
    3374
        /// Example 3: Nullable array
    3375
        /// ```
    3376
        /// use arrow::array::ListArray;
    3377
        /// use arrow::datatypes::{DataType, Int32Type};
    3378
        /// use datafusion_common::ScalarValue;
    3379
        ///
    3380
        /// let list_arr = ListArray::from_iter_primitive::<Int32Type, _, _>(vec![
    3381
        ///     Some(vec![Some(1), Some(2), Some(3)]),
    3382
        ///     None,
    3383
        ///     Some(vec![Some(4), Some(5)]),
    3384
        /// ]);
    3385
        ///
    3386
        /// // Convert the array into Scalar Values for each row
    3387
        /// let scalar_vec = ScalarValue::convert_array_to_scalar_vec(&list_arr).unwrap();
    3388
        ///
    3389
        /// let expected = vec![
    3390
        ///     Some(vec![
    3391
        ///         ScalarValue::Int32(Some(1)),
    3392
        ///         ScalarValue::Int32(Some(2)),
    3393
        ///         ScalarValue::Int32(Some(3)),
    3394
        ///     ]),
    3395
        ///     None,
    3396
        ///     Some(vec![
    3397
        ///         ScalarValue::Int32(Some(4)),
    3398
        ///         ScalarValue::Int32(Some(5)),
    3399
        ///     ]),
    3400
        /// ];
    3401
        ///
    3402
        /// assert_eq!(scalar_vec, expected);
    3403
        /// ```
    3404
    0
        pub fn convert_array_to_scalar_vec(
    3405
    0
            array: &dyn Array,
    3406
    0
        ) -> Result<Vec<Option<Vec<Self>>>> {
    3407
    0
            fn generic_collect<OffsetSize: OffsetSizeTrait>(
    3408
    0
                array: &dyn Array,
    3409
    0
            ) -> Result<Vec<Option<Vec<ScalarValue>>>> {
    3410
    0
                array
    3411
    0
                    .as_list::<OffsetSize>()
    3412
    0
                    .iter()
    3413
    0
                    .map(|nested_array| {
    3414
    0
                        nested_array
    3415
    0
                            .map(|array| {
    3416
    0
                                (0..array.len())
    3417
    0
                                    .map(|i| ScalarValue::try_from_array(&array, i))
    3418
    0
                                    .collect::<Result<Vec<_>>>()
    3419
    0
                            })
    3420
    0
                            .transpose()
    3421
    0
                    })
    3422
    0
                    .collect()
    3423
    0
            }
    3424
    3425
    0
            match array.data_type() {
    3426
    0
                DataType::List(_) => generic_collect::<i32>(array),
    3427
    0
                DataType::LargeList(_) => generic_collect::<i64>(array),
    3428
    0
                _ => _internal_err!(
    3429
    0
                    "ScalarValue::convert_array_to_scalar_vec input must be a List/LargeList type"
    3430
                ),
    3431
            }
    3432
    0
        }
    3433
    3434
        #[deprecated(
    3435
            since = "46.0.0",
    3436
            note = "This function is obsolete. Use `to_array` instead"
    3437
        )]
    3438
    0
        pub fn raw_data(&self) -> Result<ArrayRef> {
    3439
    0
            match self {
    3440
    0
                ScalarValue::List(arr) => Ok(arr.to_owned()),
    3441
    0
                _ => _internal_err!("ScalarValue is not a list"),
    3442
            }
    3443
    0
        }
    3444
    3445
        /// Converts a value in `array` at `index` into a ScalarValue
    3446
    493
        pub fn try_from_array(array: &dyn Array, index: usize) -> Result<Self> {
    3447
            // handle NULL value
    3448
    493
            if !array.is_valid(index) {
    3449
    31
                return array.data_type().try_into();
    3450
    462
            }
    3451
    3452
    462
            Ok(match array.data_type() {
    3453
    4
                DataType::Null => ScalarValue::Null,
    3454
    0
                DataType::Decimal32(precision, scale) => {
    3455
    0
                    ScalarValue::get_decimal_value_from_array(
    3456
    0
                        array, index, *precision, *scale,
    3457
    0
                    )?
    3458
                }
    3459
    0
                DataType::Decimal64(precision, scale) => {
    3460
    0
                    ScalarValue::get_decimal_value_from_array(
    3461
    0
                        array, index, *precision, *scale,
    3462
    0
                    )?
    3463
                }
    3464
    412
                DataType::Decimal128(precision, scale) => {
    3465
    412
                    ScalarValue::get_decimal_value_from_array(
    3466
    412
                        array, index, *precision, *scale,
    3467
    0
                    )?
    3468
                }
    3469
    0
                DataType::Decimal256(precision, scale) => {
    3470
    0
                    ScalarValue::get_decimal_value_from_array(
    3471
    0
                        array, index, *precision, *scale,
    3472
    0
                    )?
    3473
                }
    3474
    0
                DataType::Boolean => typed_cast!(array, index, as_boolean_array, Boolean)?,
    3475
    0
                DataType::Float64 => typed_cast!(array, index, as_float64_array, Float64)?,
    3476
    0
                DataType::Float32 => typed_cast!(array, index, as_float32_array, Float32)?,
    3477
    0
                DataType::Float16 => typed_cast!(array, index, as_float16_array, Float16)?,
    3478
    0
                DataType::UInt64 => typed_cast!(array, index, as_uint64_array, UInt64)?,
    3479
    0
                DataType::UInt32 => typed_cast!(array, index, as_uint32_array, UInt32)?,
    3480
    0
                DataType::UInt16 => typed_cast!(array, index, as_uint16_array, UInt16)?,
    3481
    0
                DataType::UInt8 => typed_cast!(array, index, as_uint8_array, UInt8)?,
    3482
    12
                DataType::Int64 => typed_cast!(array, index, as_int64_array, Int64)?0,
    3483
    2
                DataType::Int32 => typed_cast!(array, index, as_int32_array, Int32)?0,
    3484
    0
                DataType::Int16 => typed_cast!(array, index, as_int16_array, Int16)?,
    3485
    0
                DataType::Int8 => typed_cast!(array, index, as_int8_array, Int8)?,
    3486
    0
                DataType::Binary => typed_cast!(array, index, as_binary_array, Binary)?,
    3487
                DataType::LargeBinary => {
    3488
    8
                    typed_cast!(array, index, as_large_binary_array, LargeBinary)?0
    3489
                }
    3490
                DataType::BinaryView => {
    3491
    8
                    typed_cast!(array, index, as_binary_view_array, BinaryView)?0
    3492
                }
    3493
    0
                DataType::Utf8 => typed_cast!(array, index, as_string_array, Utf8)?,
    3494
                DataType::LargeUtf8 => {
    3495
    8
                    typed_cast!(array, index, as_large_string_array, LargeUtf8)?0
    3496
                }
    3497
                DataType::Utf8View => {
    3498
    8
                    typed_cast!(array, index, as_string_view_array, Utf8View)?0
    3499
                }
    3500
    0
                DataType::List(field) => {
    3501
    0
                    let list_array = array.as_list::<i32>();
    3502
    0
                    let nested_array = list_array.value(index);
    3503
                    // Produces a single element `ListArray` with the value at `index`.
    3504
    0
                    SingleRowListArrayBuilder::new(nested_array)
    3505
    0
                        .with_field(field)
    3506
    0
                        .build_list_scalar()
    3507
                }
    3508
    0
                DataType::LargeList(field) => {
    3509
    0
                    let list_array = as_large_list_array(array)?;
    3510
    0
                    let nested_array = list_array.value(index);
    3511
                    // Produces a single element `LargeListArray` with the value at `index`.
    3512
    0
                    SingleRowListArrayBuilder::new(nested_array)
    3513
    0
                        .with_field(field)
    3514
    0
                        .build_large_list_scalar()
    3515
                }
    3516
                // TODO: There is no test for FixedSizeList now, add it later
    3517
    0
                DataType::FixedSizeList(field, _) => {
    3518
    0
                    let list_array = as_fixed_size_list_array(array)?;
    3519
    0
                    let nested_array = list_array.value(index);
    3520
                    // Produces a single element `FixedSizeListArray` with the value at `index`.
    3521
    0
                    let list_size = nested_array.len();
    3522
    0
                    SingleRowListArrayBuilder::new(nested_array)
    3523
    0
                        .with_field(field)
    3524
    0
                        .build_fixed_size_list_scalar(list_size)
    3525
                }
    3526
    0
                DataType::Date32 => typed_cast!(array, index, as_date32_array, Date32)?,
    3527
    0
                DataType::Date64 => typed_cast!(array, index, as_date64_array, Date64)?,
    3528
                DataType::Time32(TimeUnit::Second) => {
    3529
    0
                    typed_cast!(array, index, as_time32_second_array, Time32Second)?
    3530
                }
    3531
                DataType::Time32(TimeUnit::Millisecond) => {
    3532
    0
                    typed_cast!(array, index, as_time32_millisecond_array, Time32Millisecond)?
    3533
                }
    3534
                DataType::Time64(TimeUnit::Microsecond) => {
    3535
    0
                    typed_cast!(array, index, as_time64_microsecond_array, Time64Microsecond)?
    3536
                }
    3537
                DataType::Time64(TimeUnit::Nanosecond) => {
    3538
    0
                    typed_cast!(array, index, as_time64_nanosecond_array, Time64Nanosecond)?
    3539
                }
    3540
    0
                DataType::Timestamp(TimeUnit::Second, tz_opt) => typed_cast_tz!(
    3541
    0
                    array,
    3542
    0
                    index,
    3543
                    as_timestamp_second_array,
    3544
                    TimestampSecond,
    3545
    0
                    tz_opt
    3546
    0
                )?,
    3547
    0
                DataType::Timestamp(TimeUnit::Millisecond, tz_opt) => typed_cast_tz!(
    3548
    0
                    array,
    3549
    0
                    index,
    3550
                    as_timestamp_millisecond_array,
    3551
                    TimestampMillisecond,
    3552
    0
                    tz_opt
    3553
    0
                )?,
    3554
    0
                DataType::Timestamp(TimeUnit::Microsecond, tz_opt) => typed_cast_tz!(
    3555
    0
                    array,
    3556
    0
                    index,
    3557
                    as_timestamp_microsecond_array,
    3558
                    TimestampMicrosecond,
    3559
    0
                    tz_opt
    3560
    0
                )?,
    3561
    0
                DataType::Timestamp(TimeUnit::Nanosecond, tz_opt) => typed_cast_tz!(
    3562
    0
                    array,
    3563
    0
                    index,
    3564
                    as_timestamp_nanosecond_array,
    3565
                    TimestampNanosecond,
    3566
    0
                    tz_opt
    3567
    0
                )?,
    3568
    0
                DataType::Dictionary(key_type, _) => {
    3569
    0
                    let (values_array, values_index) = match key_type.as_ref() {
    3570
    0
                        DataType::Int8 => get_dict_value::<Int8Type>(array, index)?,
    3571
    0
                        DataType::Int16 => get_dict_value::<Int16Type>(array, index)?,
    3572
    0
                        DataType::Int32 => get_dict_value::<Int32Type>(array, index)?,
    3573
    0
                        DataType::Int64 => get_dict_value::<Int64Type>(array, index)?,
    3574
    0
                        DataType::UInt8 => get_dict_value::<UInt8Type>(array, index)?,
    3575
    0
                        DataType::UInt16 => get_dict_value::<UInt16Type>(array, index)?,
    3576
    0
                        DataType::UInt32 => get_dict_value::<UInt32Type>(array, index)?,
    3577
    0
                        DataType::UInt64 => get_dict_value::<UInt64Type>(array, index)?,
    3578
    0
                        _ => unreachable!("Invalid dictionary keys type: {}", key_type),
    3579
                    };
    3580
                    // look up the index in the values dictionary
    3581
    0
                    let value = match values_index {
    3582
    0
                        Some(values_index) => {
    3583
    0
                            ScalarValue::try_from_array(values_array, values_index)
    3584
                        }
    3585
                        // else entry was null, so return null
    3586
    0
                        None => values_array.data_type().try_into(),
    3587
    0
                    }?;
    3588
    3589
    0
                    Self::Dictionary(key_type.clone(), Box::new(value))
    3590
                }
    3591
                DataType::Struct(_) => {
    3592
    0
                    let a = array.slice(index, 1);
    3593
    0
                    Self::Struct(Arc::new(a.as_struct().to_owned()))
    3594
                }
    3595
                DataType::FixedSizeBinary(_) => {
    3596
    0
                    let array = as_fixed_size_binary_array(array)?;
    3597
    0
                    let size = match array.data_type() {
    3598
    0
                        DataType::FixedSizeBinary(size) => *size,
    3599
    0
                        _ => unreachable!(),
    3600
                    };
    3601
                    ScalarValue::FixedSizeBinary(
    3602
    0
                        size,
    3603
    0
                        match array.is_null(index) {
    3604
    0
                            true => None,
    3605
    0
                            false => Some(array.value(index).into()),
    3606
                        },
    3607
                    )
    3608
                }
    3609
                DataType::Interval(IntervalUnit::DayTime) => {
    3610
    0
                    typed_cast!(array, index, as_interval_dt_array, IntervalDayTime)?
    3611
                }
    3612
                DataType::Interval(IntervalUnit::YearMonth) => {
    3613
    0
                    typed_cast!(array, index, as_interval_ym_array, IntervalYearMonth)?
    3614
                }
    3615
                DataType::Interval(IntervalUnit::MonthDayNano) => {
    3616
    0
                    typed_cast!(array, index, as_interval_mdn_array, IntervalMonthDayNano)?
    3617
                }
    3618
    3619
                DataType::Duration(TimeUnit::Second) => {
    3620
    0
                    typed_cast!(array, index, as_duration_second_array, DurationSecond)?
    3621
                }
    3622
    0
                DataType::Duration(TimeUnit::Millisecond) => typed_cast!(
    3623
    0
                    array,
    3624
    0
                    index,
    3625
                    as_duration_millisecond_array,
    3626
                    DurationMillisecond
    3627
    0
                )?,
    3628
    0
                DataType::Duration(TimeUnit::Microsecond) => typed_cast!(
    3629
    0
                    array,
    3630
    0
                    index,
    3631
                    as_duration_microsecond_array,
    3632
                    DurationMicrosecond
    3633
    0
                )?,
    3634
    0
                DataType::Duration(TimeUnit::Nanosecond) => typed_cast!(
    3635
    0
                    array,
    3636
    0
                    index,
    3637
                    as_duration_nanosecond_array,
    3638
                    DurationNanosecond
    3639
    0
                )?,
    3640
                DataType::Map(_, _) => {
    3641
    0
                    let a = array.slice(index, 1);
    3642
    0
                    Self::Map(Arc::new(a.as_map().to_owned()))
    3643
                }
    3644
    0
                DataType::Union(fields, mode) => {
    3645
    0
                    let array = as_union_array(array)?;
    3646
    0
                    let ti = array.type_id(index);
    3647
    0
                    let index = array.value_offset(index);
    3648
    0
                    let value = ScalarValue::try_from_array(array.child(ti), index)?;
    3649
    0
                    ScalarValue::Union(Some((ti, Box::new(value))), fields.clone(), *mode)
    3650
                }
    3651
    0
                other => {
    3652
    0
                    return _not_impl_err!(
    3653
    0
                        "Can't create a scalar from array of type \"{other:?}\""
    3654
                    );
    3655
                }
    3656
            })
    3657
    493
        }
    3658
    3659
        /// Try to parse `value` into a ScalarValue of type `target_type`
    3660
    0
        pub fn try_from_string(value: String, target_type: &DataType) -> Result<Self> {
    3661
    0
            ScalarValue::from(value).cast_to(target_type)
    3662
    0
        }
    3663
    3664
        /// Returns the Some(`&str`) representation of `ScalarValue` of logical string type
    3665
        ///
    3666
        /// Returns `None` if this `ScalarValue` is not a logical string type or the
    3667
        /// `ScalarValue` represents the `NULL` value.
    3668
        ///
    3669
        /// Note you can use [`Option::flatten`] to check for non null logical
    3670
        /// strings.
    3671
        ///
    3672
        /// For example, [`ScalarValue::Utf8`], [`ScalarValue::LargeUtf8`], and
    3673
        /// [`ScalarValue::Dictionary`] with a logical string value and store
    3674
        /// strings and can be accessed as `&str` using this method.
    3675
        ///
    3676
        /// # Example: logical strings
    3677
        /// ```
    3678
        /// # use datafusion_common::ScalarValue;
    3679
        /// /// non strings return None
    3680
        /// let scalar = ScalarValue::from(42);
    3681
        /// assert_eq!(scalar.try_as_str(), None);
    3682
        /// // Non null logical string returns Some(Some(&str))
    3683
        /// let scalar = ScalarValue::from("hello");
    3684
        /// assert_eq!(scalar.try_as_str(), Some(Some("hello")));
    3685
        /// // Null logical string returns Some(None)
    3686
        /// let scalar = ScalarValue::Utf8(None);
    3687
        /// assert_eq!(scalar.try_as_str(), Some(None));
    3688
        /// ```
    3689
        ///
    3690
        /// # Example: use [`Option::flatten`] to check for non-null logical strings
    3691
        /// ```
    3692
        /// # use datafusion_common::ScalarValue;
    3693
        /// // Non null logical string returns Some(Some(&str))
    3694
        /// let scalar = ScalarValue::from("hello");
    3695
        /// assert_eq!(scalar.try_as_str().flatten(), Some("hello"));
    3696
        /// ```
    3697
    0
        pub fn try_as_str(&self) -> Option<Option<&str>> {
    3698
    0
            let v = match self {
    3699
    0
                ScalarValue::Utf8(v) => v,
    3700
    0
                ScalarValue::LargeUtf8(v) => v,
    3701
    0
                ScalarValue::Utf8View(v) => v,
    3702
    0
                ScalarValue::Dictionary(_, v) => return v.try_as_str(),
    3703
    0
                _ => return None,
    3704
            };
    3705
    0
            Some(v.as_ref().map(|v| v.as_str()))
    3706
    0
        }
    3707
    3708
        /// Try to cast this value to a ScalarValue of type `data_type`
    3709
    0
        pub fn cast_to(&self, target_type: &DataType) -> Result<Self> {
    3710
    0
            self.cast_to_with_options(target_type, &DEFAULT_CAST_OPTIONS)
    3711
    0
        }
    3712
    3713
        /// Try to cast this value to a ScalarValue of type `data_type` with [`CastOptions`]
    3714
    483
        pub fn cast_to_with_options(
    3715
    483
            &self,
    3716
    483
            target_type: &DataType,
    3717
    483
            cast_options: &CastOptions<'static>,
    3718
    483
        ) -> Result<Self> {
    3719
    483
            let source_type = self.data_type();
    3720
    483
            if let Some(multiplier0) = date_to_timestamp_multiplier(&source_type, target_type)
    3721
            {
    3722
    0
                if let Some(value) = self.date_scalar_value_as_i64() {
    3723
    0
                    ensure_timestamp_in_bounds(value, multiplier, &source_type, target_type)?;
    3724
    0
                }
    3725
    483
            }
    3726
    3727
    483
            let scalar_array = self.to_array()?0;
    3728
    483
            let cast_arr = cast_with_options(&scalar_array, target_type, cast_options)?0;
    3729
    483
            ScalarValue::try_from_array(&cast_arr, 0)
    3730
    483
        }
    3731
    3732
    0
        fn date_scalar_value_as_i64(&self) -> Option<i64> {
    3733
    0
            match self {
    3734
    0
                ScalarValue::Date32(Some(value)) => Some(i64::from(*value)),
    3735
    0
                ScalarValue::Date64(Some(value)) => Some(*value),
    3736
    0
                _ => None,
    3737
            }
    3738
    0
        }
    3739
    3740
    0
        fn eq_array_decimal32(
    3741
    0
            array: &ArrayRef,
    3742
    0
            index: usize,
    3743
    0
            value: Option<&i32>,
    3744
    0
            precision: u8,
    3745
    0
            scale: i8,
    3746
    0
        ) -> Result<bool> {
    3747
    0
            let array = as_decimal32_array(array)?;
    3748
    0
            if array.precision() != precision || array.scale() != scale {
    3749
    0
                return Ok(false);
    3750
    0
            }
    3751
    0
            let is_null = array.is_null(index);
    3752
    0
            if let Some(v) = value {
    3753
    0
                Ok(!array.is_null(index) && array.value(index) == *v)
    3754
            } else {
    3755
    0
                Ok(is_null)
    3756
            }
    3757
    0
        }
    3758
    3759
    0
        fn eq_array_decimal64(
    3760
    0
            array: &ArrayRef,
    3761
    0
            index: usize,
    3762
    0
            value: Option<&i64>,
    3763
    0
            precision: u8,
    3764
    0
            scale: i8,
    3765
    0
        ) -> Result<bool> {
    3766
    0
            let array = as_decimal64_array(array)?;
    3767
    0
            if array.precision() != precision || array.scale() != scale {
    3768
    0
                return Ok(false);
    3769
    0
            }
    3770
    0
            let is_null = array.is_null(index);
    3771
    0
            if let Some(v) = value {
    3772
    0
                Ok(!array.is_null(index) && array.value(index) == *v)
    3773
            } else {
    3774
    0
                Ok(is_null)
    3775
            }
    3776
    0
        }
    3777
    3778
    0
        fn eq_array_decimal(
    3779
    0
            array: &ArrayRef,
    3780
    0
            index: usize,
    3781
    0
            value: Option<&i128>,
    3782
    0
            precision: u8,
    3783
    0
            scale: i8,
    3784
    0
        ) -> Result<bool> {
    3785
    0
            let array = as_decimal128_array(array)?;
    3786
    0
            if array.precision() != precision || array.scale() != scale {
    3787
    0
                return Ok(false);
    3788
    0
            }
    3789
    0
            let is_null = array.is_null(index);
    3790
    0
            if let Some(v) = value {
    3791
    0
                Ok(!array.is_null(index) && array.value(index) == *v)
    3792
            } else {
    3793
    0
                Ok(is_null)
    3794
            }
    3795
    0
        }
    3796
    3797
    0
        fn eq_array_decimal256(
    3798
    0
            array: &ArrayRef,
    3799
    0
            index: usize,
    3800
    0
            value: Option<&i256>,
    3801
    0
            precision: u8,
    3802
    0
            scale: i8,
    3803
    0
        ) -> Result<bool> {
    3804
    0
            let array = as_decimal256_array(array)?;
    3805
    0
            if array.precision() != precision || array.scale() != scale {
    3806
    0
                return Ok(false);
    3807
    0
            }
    3808
    0
            let is_null = array.is_null(index);
    3809
    0
            if let Some(v) = value {
    3810
    0
                Ok(!array.is_null(index) && array.value(index) == *v)
    3811
            } else {
    3812
    0
                Ok(is_null)
    3813
            }
    3814
    0
        }
    3815
    3816
        /// Compares a single row of array @ index for equality with self,
    3817
        /// in an optimized fashion.
    3818
        ///
    3819
        /// This method implements an optimized version of:
    3820
        ///
    3821
        /// ```text
    3822
        ///     let arr_scalar = Self::try_from_array(array, index).unwrap();
    3823
        ///     arr_scalar.eq(self)
    3824
        /// ```
    3825
        ///
    3826
        /// *Performance note*: the arrow compute kernels should be
    3827
        /// preferred over this function if at all possible as they can be
    3828
        /// vectorized and are generally much faster.
    3829
        ///
    3830
        /// This function has a few narrow use cases such as hash table key
    3831
        /// comparisons where comparing a single row at a time is necessary.
    3832
        ///
    3833
        /// # Errors
    3834
        ///
    3835
        /// Errors if
    3836
        /// - it fails to downcast `array` to the data type of `self`
    3837
        /// - `self` is a `Struct`
    3838
        ///
    3839
        /// # Panics
    3840
        ///
    3841
        /// Panics if `self` is a dictionary with invalid key type
    3842
        #[inline]
    3843
    0
        pub fn eq_array(&self, array: &ArrayRef, index: usize) -> Result<bool> {
    3844
    0
            Ok(match self {
    3845
    0
                ScalarValue::Decimal32(v, precision, scale) => {
    3846
    0
                    ScalarValue::eq_array_decimal32(
    3847
    0
                        array,
    3848
    0
                        index,
    3849
    0
                        v.as_ref(),
    3850
    0
                        *precision,
    3851
    0
                        *scale,
    3852
    0
                    )?
    3853
                }
    3854
    0
                ScalarValue::Decimal64(v, precision, scale) => {
    3855
    0
                    ScalarValue::eq_array_decimal64(
    3856
    0
                        array,
    3857
    0
                        index,
    3858
    0
                        v.as_ref(),
    3859
    0
                        *precision,
    3860
    0
                        *scale,
    3861
    0
                    )?
    3862
                }
    3863
    0
                ScalarValue::Decimal128(v, precision, scale) => {
    3864
    0
                    ScalarValue::eq_array_decimal(
    3865
    0
                        array,
    3866
    0
                        index,
    3867
    0
                        v.as_ref(),
    3868
    0
                        *precision,
    3869
    0
                        *scale,
    3870
    0
                    )?
    3871
                }
    3872
    0
                ScalarValue::Decimal256(v, precision, scale) => {
    3873
    0
                    ScalarValue::eq_array_decimal256(
    3874
    0
                        array,
    3875
    0
                        index,
    3876
    0
                        v.as_ref(),
    3877
    0
                        *precision,
    3878
    0
                        *scale,
    3879
    0
                    )?
    3880
                }
    3881
    0
                ScalarValue::Boolean(val) => {
    3882
    0
                    eq_array_primitive!(array, index, as_boolean_array, val)?
    3883
                }
    3884
    0
                ScalarValue::Float16(val) => {
    3885
    0
                    eq_array_primitive!(array, index, as_float16_array, val)?
    3886
                }
    3887
    0
                ScalarValue::Float32(val) => {
    3888
    0
                    eq_array_primitive!(array, index, as_float32_array, val)?
    3889
                }
    3890
    0
                ScalarValue::Float64(val) => {
    3891
    0
                    eq_array_primitive!(array, index, as_float64_array, val)?
    3892
                }
    3893
    0
                ScalarValue::Int8(val) => {
    3894
    0
                    eq_array_primitive!(array, index, as_int8_array, val)?
    3895
                }
    3896
    0
                ScalarValue::Int16(val) => {
    3897
    0
                    eq_array_primitive!(array, index, as_int16_array, val)?
    3898
                }
    3899
    0
                ScalarValue::Int32(val) => {
    3900
    0
                    eq_array_primitive!(array, index, as_int32_array, val)?
    3901
                }
    3902
    0
                ScalarValue::Int64(val) => {
    3903
    0
                    eq_array_primitive!(array, index, as_int64_array, val)?
    3904
                }
    3905
    0
                ScalarValue::UInt8(val) => {
    3906
    0
                    eq_array_primitive!(array, index, as_uint8_array, val)?
    3907
                }
    3908
    0
                ScalarValue::UInt16(val) => {
    3909
    0
                    eq_array_primitive!(array, index, as_uint16_array, val)?
    3910
                }
    3911
    0
                ScalarValue::UInt32(val) => {
    3912
    0
                    eq_array_primitive!(array, index, as_uint32_array, val)?
    3913
                }
    3914
    0
                ScalarValue::UInt64(val) => {
    3915
    0
                    eq_array_primitive!(array, index, as_uint64_array, val)?
    3916
                }
    3917
    0
                ScalarValue::Utf8(val) => {
    3918
    0
                    eq_array_primitive!(array, index, as_string_array, val)?
    3919
                }
    3920
    0
                ScalarValue::Utf8View(val) => {
    3921
    0
                    eq_array_primitive!(array, index, as_string_view_array, val)?
    3922
                }
    3923
    0
                ScalarValue::LargeUtf8(val) => {
    3924
    0
                    eq_array_primitive!(array, index, as_large_string_array, val)?
    3925
                }
    3926
    0
                ScalarValue::Binary(val) => {
    3927
    0
                    eq_array_primitive!(array, index, as_binary_array, val)?
    3928
                }
    3929
    0
                ScalarValue::BinaryView(val) => {
    3930
    0
                    eq_array_primitive!(array, index, as_binary_view_array, val)?
    3931
                }
    3932
    0
                ScalarValue::FixedSizeBinary(_, val) => {
    3933
    0
                    eq_array_primitive!(array, index, as_fixed_size_binary_array, val)?
    3934
                }
    3935
    0
                ScalarValue::LargeBinary(val) => {
    3936
    0
                    eq_array_primitive!(array, index, as_large_binary_array, val)?
    3937
                }
    3938
    0
                ScalarValue::List(arr) => {
    3939
    0
                    Self::eq_array_list(&(arr.to_owned() as ArrayRef), array, index)
    3940
                }
    3941
    0
                ScalarValue::LargeList(arr) => {
    3942
    0
                    Self::eq_array_list(&(arr.to_owned() as ArrayRef), array, index)
    3943
                }
    3944
    0
                ScalarValue::FixedSizeList(arr) => {
    3945
    0
                    Self::eq_array_list(&(arr.to_owned() as ArrayRef), array, index)
    3946
                }
    3947
    0
                ScalarValue::Struct(arr) => {
    3948
    0
                    Self::eq_array_list(&(arr.to_owned() as ArrayRef), array, index)
    3949
                }
    3950
    0
                ScalarValue::Map(arr) => {
    3951
    0
                    Self::eq_array_list(&(arr.to_owned() as ArrayRef), array, index)
    3952
                }
    3953
    0
                ScalarValue::Date32(val) => {
    3954
    0
                    eq_array_primitive!(array, index, as_date32_array, val)?
    3955
                }
    3956
    0
                ScalarValue::Date64(val) => {
    3957
    0
                    eq_array_primitive!(array, index, as_date64_array, val)?
    3958
                }
    3959
    0
                ScalarValue::Time32Second(val) => {
    3960
    0
                    eq_array_primitive!(array, index, as_time32_second_array, val)?
    3961
                }
    3962
    0
                ScalarValue::Time32Millisecond(val) => {
    3963
    0
                    eq_array_primitive!(array, index, as_time32_millisecond_array, val)?
    3964
                }
    3965
    0
                ScalarValue::Time64Microsecond(val) => {
    3966
    0
                    eq_array_primitive!(array, index, as_time64_microsecond_array, val)?
    3967
                }
    3968
    0
                ScalarValue::Time64Nanosecond(val) => {
    3969
    0
                    eq_array_primitive!(array, index, as_time64_nanosecond_array, val)?
    3970
                }
    3971
    0
                ScalarValue::TimestampSecond(val, _) => {
    3972
    0
                    eq_array_primitive!(array, index, as_timestamp_second_array, val)?
    3973
                }
    3974
    0
                ScalarValue::TimestampMillisecond(val, _) => {
    3975
    0
                    eq_array_primitive!(array, index, as_timestamp_millisecond_array, val)?
    3976
                }
    3977
    0
                ScalarValue::TimestampMicrosecond(val, _) => {
    3978
    0
                    eq_array_primitive!(array, index, as_timestamp_microsecond_array, val)?
    3979
                }
    3980
    0
                ScalarValue::TimestampNanosecond(val, _) => {
    3981
    0
                    eq_array_primitive!(array, index, as_timestamp_nanosecond_array, val)?
    3982
                }
    3983
    0
                ScalarValue::IntervalYearMonth(val) => {
    3984
    0
                    eq_array_primitive!(array, index, as_interval_ym_array, val)?
    3985
                }
    3986
    0
                ScalarValue::IntervalDayTime(val) => {
    3987
    0
                    eq_array_primitive!(array, index, as_interval_dt_array, val)?
    3988
                }
    3989
    0
                ScalarValue::IntervalMonthDayNano(val) => {
    3990
    0
                    eq_array_primitive!(array, index, as_interval_mdn_array, val)?
    3991
                }
    3992
    0
                ScalarValue::DurationSecond(val) => {
    3993
    0
                    eq_array_primitive!(array, index, as_duration_second_array, val)?
    3994
                }
    3995
    0
                ScalarValue::DurationMillisecond(val) => {
    3996
    0
                    eq_array_primitive!(array, index, as_duration_millisecond_array, val)?
    3997
                }
    3998
    0
                ScalarValue::DurationMicrosecond(val) => {
    3999
    0
                    eq_array_primitive!(array, index, as_duration_microsecond_array, val)?
    4000
                }
    4001
    0
                ScalarValue::DurationNanosecond(val) => {
    4002
    0
                    eq_array_primitive!(array, index, as_duration_nanosecond_array, val)?
    4003
                }
    4004
    0
                ScalarValue::Union(value, _, _) => {
    4005
    0
                    let array = as_union_array(array)?;
    4006
    0
                    let ti = array.type_id(index);
    4007
    0
                    let index = array.value_offset(index);
    4008
    0
                    if let Some((ti_v, value)) = value {
    4009
    0
                        ti_v == &ti && value.eq_array(array.child(ti), index)?
    4010
                    } else {
    4011
    0
                        array.child(ti).is_null(index)
    4012
                    }
    4013
                }
    4014
    0
                ScalarValue::Dictionary(key_type, v) => {
    4015
    0
                    let (values_array, values_index) = match key_type.as_ref() {
    4016
    0
                        DataType::Int8 => get_dict_value::<Int8Type>(array, index)?,
    4017
    0
                        DataType::Int16 => get_dict_value::<Int16Type>(array, index)?,
    4018
    0
                        DataType::Int32 => get_dict_value::<Int32Type>(array, index)?,
    4019
    0
                        DataType::Int64 => get_dict_value::<Int64Type>(array, index)?,
    4020
    0
                        DataType::UInt8 => get_dict_value::<UInt8Type>(array, index)?,
    4021
    0
                        DataType::UInt16 => get_dict_value::<UInt16Type>(array, index)?,
    4022
    0
                        DataType::UInt32 => get_dict_value::<UInt32Type>(array, index)?,
    4023
    0
                        DataType::UInt64 => get_dict_value::<UInt64Type>(array, index)?,
    4024
    0
                        _ => unreachable!("Invalid dictionary keys type: {}", key_type),
    4025
                    };
    4026
                    // was the value in the array non null?
    4027
    0
                    match values_index {
    4028
    0
                        Some(values_index) => v.eq_array(values_array, values_index)?,
    4029
    0
                        None => v.is_null(),
    4030
                    }
    4031
                }
    4032
    0
                ScalarValue::Null => array.is_null(index),
    4033
            })
    4034
    0
        }
    4035
    4036
    0
        fn eq_array_list(arr1: &ArrayRef, arr2: &ArrayRef, index: usize) -> bool {
    4037
    0
            let right = arr2.slice(index, 1);
    4038
    0
            arr1 == &right
    4039
    0
        }
    4040
    4041
        /// Compare `self` with `other` and return an `Ordering`.
    4042
        ///
    4043
        /// This is the same as [`PartialOrd`] except that it returns
    4044
        /// `Err` if the values cannot be compared, e.g., they have incompatible data types.
    4045
    0
        pub fn try_cmp(&self, other: &Self) -> Result<Ordering> {
    4046
    0
            self.partial_cmp(other).ok_or_else(|| {
    4047
    0
                _internal_datafusion_err!("Uncomparable values: {self:?}, {other:?}")
    4048
    0
            })
    4049
    0
        }
    4050
    4051
        /// Estimate size if bytes including `Self`. For values with internal containers such as `String`
    4052
        /// includes the allocated size (`capacity`) rather than the current length (`len`)
    4053
    0
        pub fn size(&self) -> usize {
    4054
    0
            size_of_val(self)
    4055
    0
                + match self {
    4056
                    ScalarValue::Null
    4057
                    | ScalarValue::Boolean(_)
    4058
                    | ScalarValue::Float16(_)
    4059
                    | ScalarValue::Float32(_)
    4060
                    | ScalarValue::Float64(_)
    4061
                    | ScalarValue::Decimal32(_, _, _)
    4062
                    | ScalarValue::Decimal64(_, _, _)
    4063
                    | ScalarValue::Decimal128(_, _, _)
    4064
                    | ScalarValue::Decimal256(_, _, _)
    4065
                    | ScalarValue::Int8(_)
    4066
                    | ScalarValue::Int16(_)
    4067
                    | ScalarValue::Int32(_)
    4068
                    | ScalarValue::Int64(_)
    4069
                    | ScalarValue::UInt8(_)
    4070
                    | ScalarValue::UInt16(_)
    4071
                    | ScalarValue::UInt32(_)
    4072
                    | ScalarValue::UInt64(_)
    4073
                    | ScalarValue::Date32(_)
    4074
                    | ScalarValue::Date64(_)
    4075
                    | ScalarValue::Time32Second(_)
    4076
                    | ScalarValue::Time32Millisecond(_)
    4077
                    | ScalarValue::Time64Microsecond(_)
    4078
                    | ScalarValue::Time64Nanosecond(_)
    4079
                    | ScalarValue::IntervalYearMonth(_)
    4080
                    | ScalarValue::IntervalDayTime(_)
    4081
                    | ScalarValue::IntervalMonthDayNano(_)
    4082
                    | ScalarValue::DurationSecond(_)
    4083
                    | ScalarValue::DurationMillisecond(_)
    4084
                    | ScalarValue::DurationMicrosecond(_)
    4085
    0
                    | ScalarValue::DurationNanosecond(_) => 0,
    4086
    0
                    ScalarValue::Utf8(s)
    4087
    0
                    | ScalarValue::LargeUtf8(s)
    4088
    0
                    | ScalarValue::Utf8View(s) => {
    4089
    0
                        s.as_ref().map(|s| s.capacity()).unwrap_or_default()
    4090
                    }
    4091
    0
                    ScalarValue::TimestampSecond(_, s)
    4092
    0
                    | ScalarValue::TimestampMillisecond(_, s)
    4093
    0
                    | ScalarValue::TimestampMicrosecond(_, s)
    4094
    0
                    | ScalarValue::TimestampNanosecond(_, s) => {
    4095
    0
                        s.as_ref().map(|s| s.len()).unwrap_or_default()
    4096
                    }
    4097
    0
                    ScalarValue::Binary(b)
    4098
    0
                    | ScalarValue::FixedSizeBinary(_, b)
    4099
    0
                    | ScalarValue::LargeBinary(b)
    4100
    0
                    | ScalarValue::BinaryView(b) => {
    4101
    0
                        b.as_ref().map(|b| b.capacity()).unwrap_or_default()
    4102
                    }
    4103
    0
                    ScalarValue::List(arr) => arr.get_array_memory_size(),
    4104
    0
                    ScalarValue::LargeList(arr) => arr.get_array_memory_size(),
    4105
    0
                    ScalarValue::FixedSizeList(arr) => arr.get_array_memory_size(),
    4106
    0
                    ScalarValue::Struct(arr) => arr.get_array_memory_size(),
    4107
    0
                    ScalarValue::Map(arr) => arr.get_array_memory_size(),
    4108
    0
                    ScalarValue::Union(vals, fields, _mode) => {
    4109
    0
                        vals.as_ref()
    4110
    0
                            .map(|(_id, sv)| sv.size() - size_of_val(sv))
    4111
    0
                            .unwrap_or_default()
    4112
                            // `fields` is boxed, so it is NOT already included in `self`
    4113
    0
                            + size_of_val(fields)
    4114
    0
                            + (size_of::<Field>() * fields.len())
    4115
    0
                            + fields.iter().map(|(_idx, field)| field.size() - size_of_val(field)).sum::<usize>()
    4116
                    }
    4117
    0
                    ScalarValue::Dictionary(dt, sv) => {
    4118
                        // `dt` and `sv` are boxed, so they are NOT already included in `self`
    4119
    0
                        dt.size() + sv.size()
    4120
                    }
    4121
                }
    4122
    0
        }
    4123
    4124
        /// Estimates [size](Self::size) of [`Vec`] in bytes.
    4125
        ///
    4126
        /// Includes the size of the [`Vec`] container itself.
    4127
    0
        pub fn size_of_vec(vec: &Vec<Self>) -> usize {
    4128
    0
            size_of_val(vec)
    4129
    0
                + (size_of::<ScalarValue>() * vec.capacity())
    4130
    0
                + vec
    4131
    0
                    .iter()
    4132
    0
                    .map(|sv| sv.size() - size_of_val(sv))
    4133
    0
                    .sum::<usize>()
    4134
    0
        }
    4135
    4136
        /// Estimates [size](Self::size) of [`VecDeque`] in bytes.
    4137
        ///
    4138
        /// Includes the size of the [`VecDeque`] container itself.
    4139
    0
        pub fn size_of_vec_deque(vec_deque: &VecDeque<Self>) -> usize {
    4140
    0
            size_of_val(vec_deque)
    4141
    0
                + (size_of::<ScalarValue>() * vec_deque.capacity())
    4142
    0
                + vec_deque
    4143
    0
                    .iter()
    4144
    0
                    .map(|sv| sv.size() - size_of_val(sv))
    4145
    0
                    .sum::<usize>()
    4146
    0
        }
    4147
    4148
        /// Estimates [size](Self::size) of [`HashSet`] in bytes.
    4149
        ///
    4150
        /// Includes the size of the [`HashSet`] container itself.
    4151
    0
        pub fn size_of_hashset<S>(set: &HashSet<Self, S>) -> usize {
    4152
    0
            size_of_val(set)
    4153
    0
                + (size_of::<ScalarValue>() * set.capacity())
    4154
    0
                + set
    4155
    0
                    .iter()
    4156
    0
                    .map(|sv| sv.size() - size_of_val(sv))
    4157
    0
                    .sum::<usize>()
    4158
    0
        }
    4159
    4160
        /// Compacts the allocation referenced by `self` to the minimum, copying the data if
    4161
        /// necessary.
    4162
        ///
    4163
        /// This can be relevant when `self` is a list or contains a list as a nested value, as
    4164
        /// a single list holds an Arc to its entire original array buffer.
    4165
    0
        pub fn compact(&mut self) {
    4166
    0
            match self {
    4167
                ScalarValue::Null
    4168
                | ScalarValue::Boolean(_)
    4169
                | ScalarValue::Float16(_)
    4170
                | ScalarValue::Float32(_)
    4171
                | ScalarValue::Float64(_)
    4172
                | ScalarValue::Decimal32(_, _, _)
    4173
                | ScalarValue::Decimal64(_, _, _)
    4174
                | ScalarValue::Decimal128(_, _, _)
    4175
                | ScalarValue::Decimal256(_, _, _)
    4176
                | ScalarValue::Int8(_)
    4177
                | ScalarValue::Int16(_)
    4178
                | ScalarValue::Int32(_)
    4179
                | ScalarValue::Int64(_)
    4180
                | ScalarValue::UInt8(_)
    4181
                | ScalarValue::UInt16(_)
    4182
                | ScalarValue::UInt32(_)
    4183
                | ScalarValue::UInt64(_)
    4184
                | ScalarValue::Date32(_)
    4185
                | ScalarValue::Date64(_)
    4186
                | ScalarValue::Time32Second(_)
    4187
                | ScalarValue::Time32Millisecond(_)
    4188
                | ScalarValue::Time64Microsecond(_)
    4189
                | ScalarValue::Time64Nanosecond(_)
    4190
                | ScalarValue::IntervalYearMonth(_)
    4191
                | ScalarValue::IntervalDayTime(_)
    4192
                | ScalarValue::IntervalMonthDayNano(_)
    4193
                | ScalarValue::DurationSecond(_)
    4194
                | ScalarValue::DurationMillisecond(_)
    4195
                | ScalarValue::DurationMicrosecond(_)
    4196
                | ScalarValue::DurationNanosecond(_)
    4197
                | ScalarValue::Utf8(_)
    4198
                | ScalarValue::LargeUtf8(_)
    4199
                | ScalarValue::Utf8View(_)
    4200
                | ScalarValue::TimestampSecond(_, _)
    4201
                | ScalarValue::TimestampMillisecond(_, _)
    4202
                | ScalarValue::TimestampMicrosecond(_, _)
    4203
                | ScalarValue::TimestampNanosecond(_, _)
    4204
                | ScalarValue::Binary(_)
    4205
                | ScalarValue::FixedSizeBinary(_, _)
    4206
                | ScalarValue::LargeBinary(_)
    4207
    0
                | ScalarValue::BinaryView(_) => (),
    4208
    0
                ScalarValue::FixedSizeList(arr) => {
    4209
    0
                    let array = copy_array_data(&arr.to_data());
    4210
    0
                    *Arc::make_mut(arr) = FixedSizeListArray::from(array);
    4211
    0
                }
    4212
    0
                ScalarValue::List(arr) => {
    4213
    0
                    let array = copy_array_data(&arr.to_data());
    4214
    0
                    *Arc::make_mut(arr) = ListArray::from(array);
    4215
    0
                }
    4216
    0
                ScalarValue::LargeList(arr) => {
    4217
    0
                    let array = copy_array_data(&arr.to_data());
    4218
    0
                    *Arc::make_mut(arr) = LargeListArray::from(array)
    4219
                }
    4220
    0
                ScalarValue::Struct(arr) => {
    4221
    0
                    let array = copy_array_data(&arr.to_data());
    4222
    0
                    *Arc::make_mut(arr) = StructArray::from(array);
    4223
    0
                }
    4224
    0
                ScalarValue::Map(arr) => {
    4225
    0
                    let array = copy_array_data(&arr.to_data());
    4226
    0
                    *Arc::make_mut(arr) = MapArray::from(array);
    4227
    0
                }
    4228
    0
                ScalarValue::Union(val, _, _) => {
    4229
    0
                    if let Some((_, value)) = val.as_mut() {
    4230
    0
                        value.compact();
    4231
    0
                    }
    4232
                }
    4233
    0
                ScalarValue::Dictionary(_, value) => {
    4234
    0
                    value.compact();
    4235
    0
                }
    4236
            }
    4237
    0
        }
    4238
    4239
        /// Compacts ([ScalarValue::compact]) the current [ScalarValue] and returns it.
    4240
    0
        pub fn compacted(mut self) -> Self {
    4241
    0
            self.compact();
    4242
    0
            self
    4243
    0
        }
    4244
    4245
        /// Returns the minimum value for the given numeric `DataType`.
    4246
        ///
    4247
        /// This function returns the smallest representable value for numeric
    4248
        /// and temporal data types. For non-numeric types, it returns `None`.
    4249
        ///
    4250
        /// # Supported Types
    4251
        ///
    4252
        /// - **Integer types**: `i8::MIN`, `i16::MIN`, etc.
    4253
        /// - **Unsigned types**: Always 0 (`u8::MIN`, `u16::MIN`, etc.)
    4254
        /// - **Float types**: Negative infinity (IEEE 754)
    4255
        /// - **Decimal types**: Smallest value based on precision
    4256
        /// - **Temporal types**: Minimum timestamp/date values
    4257
        /// - **Time types**: 0 (midnight)
    4258
        /// - **Duration types**: `i64::MIN`
    4259
    0
        pub fn min(datatype: &DataType) -> Option<ScalarValue> {
    4260
    0
            match datatype {
    4261
    0
                DataType::Int8 => Some(ScalarValue::Int8(Some(i8::MIN))),
    4262
    0
                DataType::Int16 => Some(ScalarValue::Int16(Some(i16::MIN))),
    4263
    0
                DataType::Int32 => Some(ScalarValue::Int32(Some(i32::MIN))),
    4264
    0
                DataType::Int64 => Some(ScalarValue::Int64(Some(i64::MIN))),
    4265
    0
                DataType::UInt8 => Some(ScalarValue::UInt8(Some(u8::MIN))),
    4266
    0
                DataType::UInt16 => Some(ScalarValue::UInt16(Some(u16::MIN))),
    4267
    0
                DataType::UInt32 => Some(ScalarValue::UInt32(Some(u32::MIN))),
    4268
    0
                DataType::UInt64 => Some(ScalarValue::UInt64(Some(u64::MIN))),
    4269
    0
                DataType::Float16 => Some(ScalarValue::Float16(Some(f16::NEG_INFINITY))),
    4270
    0
                DataType::Float32 => Some(ScalarValue::Float32(Some(f32::NEG_INFINITY))),
    4271
    0
                DataType::Float64 => Some(ScalarValue::Float64(Some(f64::NEG_INFINITY))),
    4272
    0
                DataType::Decimal128(precision, scale) => {
    4273
                    // For decimal, min is -10^(precision-scale) + 10^(-scale)
    4274
                    // But for simplicity, we use the minimum i128 value that fits the precision
    4275
    0
                    let max_digits = 10_i128.pow(*precision as u32) - 1;
    4276
    0
                    Some(ScalarValue::Decimal128(
    4277
    0
                        Some(-max_digits),
    4278
    0
                        *precision,
    4279
    0
                        *scale,
    4280
    0
                    ))
    4281
                }
    4282
    0
                DataType::Decimal256(precision, scale) => {
    4283
                    // Similar to Decimal128 but with i256
    4284
                    // For now, use a large negative value
    4285
    0
                    let max_digits = i256::from_i128(10_i128)
    4286
    0
                        .checked_pow(*precision as u32)
    4287
    0
                        .and_then(|v| v.checked_sub(i256::from_i128(1)))
    4288
    0
                        .unwrap_or(i256::MAX);
    4289
    0
                    Some(ScalarValue::Decimal256(
    4290
    0
                        Some(max_digits.neg_wrapping()),
    4291
    0
                        *precision,
    4292
    0
                        *scale,
    4293
    0
                    ))
    4294
                }
    4295
    0
                DataType::Date32 => Some(ScalarValue::Date32(Some(i32::MIN))),
    4296
    0
                DataType::Date64 => Some(ScalarValue::Date64(Some(i64::MIN))),
    4297
                DataType::Time32(TimeUnit::Second) => {
    4298
    0
                    Some(ScalarValue::Time32Second(Some(0)))
    4299
                }
    4300
                DataType::Time32(TimeUnit::Millisecond) => {
    4301
    0
                    Some(ScalarValue::Time32Millisecond(Some(0)))
    4302
                }
    4303
                DataType::Time64(TimeUnit::Microsecond) => {
    4304
    0
                    Some(ScalarValue::Time64Microsecond(Some(0)))
    4305
                }
    4306
                DataType::Time64(TimeUnit::Nanosecond) => {
    4307
    0
                    Some(ScalarValue::Time64Nanosecond(Some(0)))
    4308
                }
    4309
    0
                DataType::Timestamp(unit, tz) => match unit {
    4310
                    TimeUnit::Second => {
    4311
    0
                        Some(ScalarValue::TimestampSecond(Some(i64::MIN), tz.clone()))
    4312
                    }
    4313
    0
                    TimeUnit::Millisecond => Some(ScalarValue::TimestampMillisecond(
    4314
    0
                        Some(i64::MIN),
    4315
    0
                        tz.clone(),
    4316
    0
                    )),
    4317
    0
                    TimeUnit::Microsecond => Some(ScalarValue::TimestampMicrosecond(
    4318
    0
                        Some(i64::MIN),
    4319
    0
                        tz.clone(),
    4320
    0
                    )),
    4321
                    TimeUnit::Nanosecond => {
    4322
    0
                        Some(ScalarValue::TimestampNanosecond(Some(i64::MIN), tz.clone()))
    4323
                    }
    4324
                },
    4325
    0
                DataType::Duration(unit) => match unit {
    4326
    0
                    TimeUnit::Second => Some(ScalarValue::DurationSecond(Some(i64::MIN))),
    4327
                    TimeUnit::Millisecond => {
    4328
    0
                        Some(ScalarValue::DurationMillisecond(Some(i64::MIN)))
    4329
                    }
    4330
                    TimeUnit::Microsecond => {
    4331
    0
                        Some(ScalarValue::DurationMicrosecond(Some(i64::MIN)))
    4332
                    }
    4333
                    TimeUnit::Nanosecond => {
    4334
    0
                        Some(ScalarValue::DurationNanosecond(Some(i64::MIN)))
    4335
                    }
    4336
                },
    4337
    0
                _ => None,
    4338
            }
    4339
    0
        }
    4340
    4341
        /// Returns the maximum value for the given numeric `DataType`.
    4342
        ///
    4343
        /// This function returns the largest representable value for numeric
    4344
        /// and temporal data types. For non-numeric types, it returns `None`.
    4345
        ///
    4346
        /// # Supported Types
    4347
        ///
    4348
        /// - **Integer types**: `i8::MAX`, `i16::MAX`, etc.
    4349
        /// - **Unsigned types**: `u8::MAX`, `u16::MAX`, etc.
    4350
        /// - **Float types**: Positive infinity (IEEE 754)
    4351
        /// - **Decimal types**: Largest value based on precision
    4352
        /// - **Temporal types**: Maximum timestamp/date values
    4353
        /// - **Time types**: Maximum time in the day (1 day - 1 unit)
    4354
        /// - **Duration types**: `i64::MAX`
    4355
    0
        pub fn max(datatype: &DataType) -> Option<ScalarValue> {
    4356
    0
            match datatype {
    4357
    0
                DataType::Int8 => Some(ScalarValue::Int8(Some(i8::MAX))),
    4358
    0
                DataType::Int16 => Some(ScalarValue::Int16(Some(i16::MAX))),
    4359
    0
                DataType::Int32 => Some(ScalarValue::Int32(Some(i32::MAX))),
    4360
    0
                DataType::Int64 => Some(ScalarValue::Int64(Some(i64::MAX))),
    4361
    0
                DataType::UInt8 => Some(ScalarValue::UInt8(Some(u8::MAX))),
    4362
    0
                DataType::UInt16 => Some(ScalarValue::UInt16(Some(u16::MAX))),
    4363
    0
                DataType::UInt32 => Some(ScalarValue::UInt32(Some(u32::MAX))),
    4364
    0
                DataType::UInt64 => Some(ScalarValue::UInt64(Some(u64::MAX))),
    4365
    0
                DataType::Float16 => Some(ScalarValue::Float16(Some(f16::INFINITY))),
    4366
    0
                DataType::Float32 => Some(ScalarValue::Float32(Some(f32::INFINITY))),
    4367
    0
                DataType::Float64 => Some(ScalarValue::Float64(Some(f64::INFINITY))),
    4368
    0
                DataType::Decimal128(precision, scale) => {
    4369
                    // For decimal, max is 10^(precision-scale) - 10^(-scale)
    4370
                    // But for simplicity, we use the maximum i128 value that fits the precision
    4371
    0
                    let max_digits = 10_i128.pow(*precision as u32) - 1;
    4372
    0
                    Some(ScalarValue::Decimal128(
    4373
    0
                        Some(max_digits),
    4374
    0
                        *precision,
    4375
    0
                        *scale,
    4376
    0
                    ))
    4377
                }
    4378
    0
                DataType::Decimal256(precision, scale) => {
    4379
                    // Similar to Decimal128 but with i256
    4380
    0
                    let max_digits = i256::from_i128(10_i128)
    4381
    0
                        .checked_pow(*precision as u32)
    4382
    0
                        .and_then(|v| v.checked_sub(i256::from_i128(1)))
    4383
    0
                        .unwrap_or(i256::MAX);
    4384
    0
                    Some(ScalarValue::Decimal256(
    4385
    0
                        Some(max_digits),
    4386
    0
                        *precision,
    4387
    0
                        *scale,
    4388
    0
                    ))
    4389
                }
    4390
    0
                DataType::Date32 => Some(ScalarValue::Date32(Some(i32::MAX))),
    4391
    0
                DataType::Date64 => Some(ScalarValue::Date64(Some(i64::MAX))),
    4392
                DataType::Time32(TimeUnit::Second) => {
    4393
                    // 86399 seconds = 23:59:59
    4394
    0
                    Some(ScalarValue::Time32Second(Some(86_399)))
    4395
                }
    4396
                DataType::Time32(TimeUnit::Millisecond) => {
    4397
                    // 86_399_999 milliseconds = 23:59:59.999
    4398
    0
                    Some(ScalarValue::Time32Millisecond(Some(86_399_999)))
    4399
                }
    4400
                DataType::Time64(TimeUnit::Microsecond) => {
    4401
                    // 86_399_999_999 microseconds = 23:59:59.999999
    4402
    0
                    Some(ScalarValue::Time64Microsecond(Some(86_399_999_999)))
    4403
                }
    4404
                DataType::Time64(TimeUnit::Nanosecond) => {
    4405
                    // 86_399_999_999_999 nanoseconds = 23:59:59.999999999
    4406
    0
                    Some(ScalarValue::Time64Nanosecond(Some(86_399_999_999_999)))
    4407
                }
    4408
    0
                DataType::Timestamp(unit, tz) => match unit {
    4409
                    TimeUnit::Second => {
    4410
    0
                        Some(ScalarValue::TimestampSecond(Some(i64::MAX), tz.clone()))
    4411
                    }
    4412
    0
                    TimeUnit::Millisecond => Some(ScalarValue::TimestampMillisecond(
    4413
    0
                        Some(i64::MAX),
    4414
    0
                        tz.clone(),
    4415
    0
                    )),
    4416
    0
                    TimeUnit::Microsecond => Some(ScalarValue::TimestampMicrosecond(
    4417
    0
                        Some(i64::MAX),
    4418
    0
                        tz.clone(),
    4419
    0
                    )),
    4420
                    TimeUnit::Nanosecond => {
    4421
    0
                        Some(ScalarValue::TimestampNanosecond(Some(i64::MAX), tz.clone()))
    4422
                    }
    4423
                },
    4424
    0
                DataType::Duration(unit) => match unit {
    4425
    0
                    TimeUnit::Second => Some(ScalarValue::DurationSecond(Some(i64::MAX))),
    4426
                    TimeUnit::Millisecond => {
    4427
    0
                        Some(ScalarValue::DurationMillisecond(Some(i64::MAX)))
    4428
                    }
    4429
                    TimeUnit::Microsecond => {
    4430
    0
                        Some(ScalarValue::DurationMicrosecond(Some(i64::MAX)))
    4431
                    }
    4432
                    TimeUnit::Nanosecond => {
    4433
    0
                        Some(ScalarValue::DurationNanosecond(Some(i64::MAX)))
    4434
                    }
    4435
                },
    4436
    0
                _ => None,
    4437
            }
    4438
    0
        }
    4439
    4440
        /// A thin wrapper on Arrow's validation that throws internal error if validation
    4441
        /// fails.
    4442
    0
        fn validate_decimal_or_internal_err<T: DecimalType>(
    4443
    0
            precision: u8,
    4444
    0
            scale: i8,
    4445
    0
        ) -> Result<()> {
    4446
    0
            validate_decimal_precision_and_scale::<T>(precision, scale).map_err(|err| {
    4447
    0
                _internal_datafusion_err!(
    4448
    0
                    "Decimal precision/scale invariant violated \
    4449
    0
                     (precision={precision}, scale={scale}): {err}"
    4450
                )
    4451
    0
            })
    4452
    0
        }
    4453
    }
    4454
    4455
    /// Compacts the data of an `ArrayData` into a new `ArrayData`.
    4456
    ///
    4457
    /// This is useful when you want to minimize the memory footprint of an
    4458
    /// `ArrayData`. For example, the value returned by [`Array::slice`] still
    4459
    /// points at the same underlying data buffers as the original array, which may
    4460
    /// hold many more values. Calling `copy_array_data` on the sliced array will
    4461
    /// create a new, smaller, `ArrayData` that only contains the data for the
    4462
    /// sliced array.
    4463
    ///
    4464
    /// # Example
    4465
    /// ```
    4466
    /// # use arrow::array::{make_array, Array, Int32Array};
    4467
    /// use datafusion_common::scalar::copy_array_data;
    4468
    /// let array = Int32Array::from_iter_values(0..8192);
    4469
    /// // Take only the first 2 elements
    4470
    /// let sliced_array = array.slice(0, 2);
    4471
    /// // The memory footprint of `sliced_array` is close to 8192 * 4 bytes
    4472
    /// assert_eq!(32864, sliced_array.get_array_memory_size());
    4473
    /// // however, we can copy the data to a new `ArrayData`
    4474
    /// let new_array = make_array(copy_array_data(&sliced_array.into_data()));
    4475
    /// // The memory footprint of `new_array` is now only 2 * 4 bytes
    4476
    /// // and overhead:
    4477
    /// assert_eq!(160, new_array.get_array_memory_size());
    4478
    /// ```
    4479
    ///
    4480
    /// See also [`ScalarValue::compact`] which applies to `ScalarValue` instances
    4481
    /// as necessary.
    4482
    0
    pub fn copy_array_data(src_data: &ArrayData) -> ArrayData {
    4483
    0
        let mut copy = MutableArrayData::new(vec![&src_data], true, src_data.len());
    4484
    0
        copy.extend(0, 0, src_data.len());
    4485
    0
        copy.freeze()
    4486
    0
    }
    4487
    4488
    macro_rules! impl_scalar {
    4489
        ($ty:ty, $scalar:tt) => {
    4490
            impl From<$ty> for ScalarValue {
    4491
    2
                fn from(value: $ty) -> Self {
    4492
    2
                    ScalarValue::$scalar(Some(value))
    4493
    2
                }
    4494
            }
    4495
    4496
            impl From<Option<$ty>> for ScalarValue {
    4497
    0
                fn from(value: Option<$ty>) -> Self {
    4498
    0
                    ScalarValue::$scalar(value)
    4499
    0
                }
    4500
            }
    4501
        };
    4502
    }
    4503
    4504
    impl_scalar!(f64, Float64);
    4505
    impl_scalar!(f32, Float32);
    4506
    impl_scalar!(f16, Float16);
    4507
    impl_scalar!(i8, Int8);
    4508
    impl_scalar!(i16, Int16);
    4509
    impl_scalar!(i32, Int32);
    4510
    impl_scalar!(i64, Int64);
    4511
    impl_scalar!(bool, Boolean);
    4512
    impl_scalar!(u8, UInt8);
    4513
    impl_scalar!(u16, UInt16);
    4514
    impl_scalar!(u32, UInt32);
    4515
    impl_scalar!(u64, UInt64);
    4516
    4517
    impl From<&str> for ScalarValue {
    4518
    50
        fn from(value: &str) -> Self {
    4519
    50
            Some(value).into()
    4520
    50
        }
    4521
    }
    4522
    4523
    impl From<Option<&str>> for ScalarValue {
    4524
    50
        fn from(value: Option<&str>) -> Self {
    4525
    50
            let value = value.map(|s| s.to_string());
    4526
    50
            value.into()
    4527
    50
        }
    4528
    }
    4529
    4530
    /// Wrapper to create ScalarValue::Struct for convenience
    4531
    impl From<Vec<(&str, ScalarValue)>> for ScalarValue {
    4532
    0
        fn from(value: Vec<(&str, ScalarValue)>) -> Self {
    4533
    0
            value
    4534
    0
                .into_iter()
    4535
    0
                .fold(ScalarStructBuilder::new(), |builder, (name, value)| {
    4536
    0
                    builder.with_name_and_scalar(name, value)
    4537
    0
                })
    4538
    0
                .build()
    4539
    0
                .unwrap()
    4540
    0
        }
    4541
    }
    4542
    4543
    impl FromStr for ScalarValue {
    4544
        type Err = Infallible;
    4545
    4546
    0
        fn from_str(s: &str) -> Result<Self, Self::Err> {
    4547
    0
            Ok(s.into())
    4548
    0
        }
    4549
    }
    4550
    4551
    impl From<String> for ScalarValue {
    4552
    4
        fn from(value: String) -> Self {
    4553
    4
            Some(value).into()
    4554
    4
        }
    4555
    }
    4556
    4557
    impl From<Option<String>> for ScalarValue {
    4558
    54
        fn from(value: Option<String>) -> Self {
    4559
    54
            ScalarValue::Utf8(value)
    4560
    54
        }
    4561
    }
    4562
    4563
    macro_rules! impl_try_from {
    4564
        ($SCALAR:ident, $NATIVE:ident) => {
    4565
            impl TryFrom<ScalarValue> for $NATIVE {
    4566
                type Error = DataFusionError;
    4567
    4568
    0
                fn try_from(value: ScalarValue) -> Result<Self> {
    4569
    0
                    match value {
    4570
    0
                        ScalarValue::$SCALAR(Some(inner_value)) => Ok(inner_value),
    4571
    0
                        _ => _internal_err!(
    4572
    0
                            "Cannot convert {:?} to {}",
    4573
                            value,
    4574
    0
                            std::any::type_name::<Self>()
    4575
                        ),
    4576
                    }
    4577
    0
                }
    4578
            }
    4579
        };
    4580
    }
    4581
    4582
    impl_try_from!(Int8, i8);
    4583
    impl_try_from!(Int16, i16);
    4584
    4585
    // special implementation for i32 because of Date32 and Time32
    4586
    impl TryFrom<ScalarValue> for i32 {
    4587
        type Error = DataFusionError;
    4588
    4589
    0
        fn try_from(value: ScalarValue) -> Result<Self> {
    4590
    0
            match value {
    4591
    0
                ScalarValue::Int32(Some(inner_value))
    4592
    0
                | ScalarValue::Date32(Some(inner_value))
    4593
    0
                | ScalarValue::Time32Second(Some(inner_value))
    4594
    0
                | ScalarValue::Time32Millisecond(Some(inner_value)) => Ok(inner_value),
    4595
    0
                _ => _internal_err!(
    4596
    0
                    "Cannot convert {:?} to {}",
    4597
                    value,
    4598
    0
                    std::any::type_name::<Self>()
    4599
                ),
    4600
            }
    4601
    0
        }
    4602
    }
    4603
    4604
    // special implementation for i64 because of Date64, Time64 and Timestamp
    4605
    impl TryFrom<ScalarValue> for i64 {
    4606
        type Error = DataFusionError;
    4607
    4608
    0
        fn try_from(value: ScalarValue) -> Result<Self> {
    4609
    0
            match value {
    4610
    0
                ScalarValue::Int64(Some(inner_value))
    4611
    0
                | ScalarValue::Date64(Some(inner_value))
    4612
    0
                | ScalarValue::Time64Microsecond(Some(inner_value))
    4613
    0
                | ScalarValue::Time64Nanosecond(Some(inner_value))
    4614
    0
                | ScalarValue::TimestampNanosecond(Some(inner_value), _)
    4615
    0
                | ScalarValue::TimestampMicrosecond(Some(inner_value), _)
    4616
    0
                | ScalarValue::TimestampMillisecond(Some(inner_value), _)
    4617
    0
                | ScalarValue::TimestampSecond(Some(inner_value), _) => Ok(inner_value),
    4618
    0
                _ => _internal_err!(
    4619
    0
                    "Cannot convert {:?} to {}",
    4620
                    value,
    4621
    0
                    std::any::type_name::<Self>()
    4622
                ),
    4623
            }
    4624
    0
        }
    4625
    }
    4626
    4627
    // special implementation for i128 because of Decimal128
    4628
    impl TryFrom<ScalarValue> for i128 {
    4629
        type Error = DataFusionError;
    4630
    4631
    0
        fn try_from(value: ScalarValue) -> Result<Self> {
    4632
    0
            match value {
    4633
    0
                ScalarValue::Decimal128(Some(inner_value), _, _) => Ok(inner_value),
    4634
    0
                _ => _internal_err!(
    4635
    0
                    "Cannot convert {:?} to {}",
    4636
                    value,
    4637
    0
                    std::any::type_name::<Self>()
    4638
                ),
    4639
            }
    4640
    0
        }
    4641
    }
    4642
    4643
    // special implementation for i256 because of Decimal128
    4644
    impl TryFrom<ScalarValue> for i256 {
    4645
        type Error = DataFusionError;
    4646
    4647
    0
        fn try_from(value: ScalarValue) -> Result<Self> {
    4648
    0
            match value {
    4649
    0
                ScalarValue::Decimal256(Some(inner_value), _, _) => Ok(inner_value),
    4650
    0
                _ => _internal_err!(
    4651
    0
                    "Cannot convert {:?} to {}",
    4652
                    value,
    4653
    0
                    std::any::type_name::<Self>()
    4654
                ),
    4655
            }
    4656
    0
        }
    4657
    }
    4658
    4659
    impl_try_from!(UInt8, u8);
    4660
    impl_try_from!(UInt16, u16);
    4661
    impl_try_from!(UInt32, u32);
    4662
    impl_try_from!(UInt64, u64);
    4663
    impl_try_from!(Float16, f16);
    4664
    impl_try_from!(Float32, f32);
    4665
    impl_try_from!(Float64, f64);
    4666
    impl_try_from!(Boolean, bool);
    4667
    4668
    impl TryFrom<DataType> for ScalarValue {
    4669
        type Error = DataFusionError;
    4670
    4671
        /// Create a Null instance of ScalarValue for this datatype
    4672
    0
        fn try_from(datatype: DataType) -> Result<Self> {
    4673
    0
            (&datatype).try_into()
    4674
    0
        }
    4675
    }
    4676
    4677
    impl TryFrom<&DataType> for ScalarValue {
    4678
        type Error = DataFusionError;
    4679
    4680
        /// Create a Null instance of ScalarValue for this datatype
    4681
    31
        fn try_from(data_type: &DataType) -> Result<Self> {
    4682
    31
            Self::try_new_null(data_type)
    4683
    31
        }
    4684
    }
    4685
    4686
    macro_rules! format_option {
    4687
        ($F:expr, $EXPR:expr) => {{
    4688
            match $EXPR {
    4689
                Some(e) => write!($F, "{e}"),
    4690
                None => write!($F, "NULL"),
    4691
            }
    4692
        }};
    4693
    }
    4694
    4695
    // Implement Display trait for ScalarValue
    4696
    //
    4697
    // # Panics
    4698
    //
    4699
    // Panics if there is an error when creating a visual representation of columns via `arrow::util::pretty`
    4700
    impl fmt::Display for ScalarValue {
    4701
    22
        fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
    4702
    22
            match self {
    4703
    0
                ScalarValue::Decimal32(v, p, s) => {
    4704
    0
                    write!(f, "{v:?},{p:?},{s:?}")?;
    4705
                }
    4706
    0
                ScalarValue::Decimal64(v, p, s) => {
    4707
    0
                    write!(f, "{v:?},{p:?},{s:?}")?;
    4708
                }
    4709
    0
                ScalarValue::Decimal128(v, p, s) => {
    4710
    0
                    write!(f, "{v:?},{p:?},{s:?}")?;
    4711
                }
    4712
    0
                ScalarValue::Decimal256(v, p, s) => {
    4713
    0
                    write!(f, "{v:?},{p:?},{s:?}")?;
    4714
                }
    4715
    0
                ScalarValue::Boolean(e) => format_option!(f, e)?,
    4716
    0
                ScalarValue::Float16(e) => format_option!(f, e)?,
    4717
    0
                ScalarValue::Float32(e) => format_option!(f, e)?,
    4718
    0
                ScalarValue::Float64(e) => format_option!(f, e)?,
    4719
    0
                ScalarValue::Int8(e) => format_option!(f, e)?,
    4720
    0
                ScalarValue::Int16(e) => format_option!(f, e)?,
    4721
    0
                ScalarValue::Int32(e) => format_option!(f, e)?,
    4722
    0
                ScalarValue::Int64(e) => format_option!(f, e)?,
    4723
    0
                ScalarValue::UInt8(e) => format_option!(f, e)?,
    4724
    0
                ScalarValue::UInt16(e) => format_option!(f, e)?,
    4725
    0
                ScalarValue::UInt32(e) => format_option!(f, e)?,
    4726
    0
                ScalarValue::UInt64(e) => format_option!(f, e)?,
    4727
    0
                ScalarValue::TimestampSecond(e, _) => format_option!(f, e)?,
    4728
    0
                ScalarValue::TimestampMillisecond(e, _) => format_option!(f, e)?,
    4729
    0
                ScalarValue::TimestampMicrosecond(e, _) => format_option!(f, e)?,
    4730
    0
                ScalarValue::TimestampNanosecond(e, _) => format_option!(f, e)?,
    4731
    20
                ScalarValue::Utf8(e)
    4732
    0
                | ScalarValue::LargeUtf8(e)
    4733
    20
                | ScalarValue::Utf8View(e0) => format_option!16(f4, e)?0,
    4734
    0
                ScalarValue::Binary(e)
    4735
    0
                | ScalarValue::FixedSizeBinary(_, e)
    4736
    0
                | ScalarValue::LargeBinary(e)
    4737
    0
                | ScalarValue::BinaryView(e) => match e {
    4738
    0
                    Some(bytes) => {
    4739
                        // print up to first 10 bytes, with trailing ... if needed
    4740
    0
                        for b in bytes.iter().take(10) {
    4741
    0
                            write!(f, "{b:02X}")?;
    4742
                        }
    4743
    0
                        if bytes.len() > 10 {
    4744
    0
                            write!(f, "...")?;
    4745
    0
                        }
    4746
                    }
    4747
    0
                    None => write!(f, "NULL")?,
    4748
                },
    4749
    0
                ScalarValue::List(arr) => fmt_list(arr.as_ref(), f)?,
    4750
    0
                ScalarValue::LargeList(arr) => fmt_list(arr.as_ref(), f)?,
    4751
    0
                ScalarValue::FixedSizeList(arr) => fmt_list(arr.as_ref(), f)?,
    4752
    0
                ScalarValue::Date32(e) => format_option!(
    4753
    0
                    f,
    4754
    0
                    e.map(|v| {
    4755
    0
                        let epoch = NaiveDate::from_ymd_opt(1970, 1, 1).unwrap();
    4756
    0
                        match epoch.checked_add_signed(Duration::try_days(v as i64).unwrap())
    4757
                        {
    4758
    0
                            Some(date) => date.to_string(),
    4759
    0
                            None => "".to_string(),
    4760
                        }
    4761
    0
                    })
    4762
    0
                )?,
    4763
    0
                ScalarValue::Date64(e) => format_option!(
    4764
    0
                    f,
    4765
    0
                    e.map(|v| {
    4766
    0
                        let epoch = NaiveDate::from_ymd_opt(1970, 1, 1).unwrap();
    4767
    0
                        match epoch.checked_add_signed(Duration::try_milliseconds(v).unwrap())
    4768
                        {
    4769
    0
                            Some(date) => date.to_string(),
    4770
    0
                            None => "".to_string(),
    4771
                        }
    4772
    0
                    })
    4773
    0
                )?,
    4774
    0
                ScalarValue::Time32Second(e) => format_option!(f, e)?,
    4775
    0
                ScalarValue::Time32Millisecond(e) => format_option!(f, e)?,
    4776
    0
                ScalarValue::Time64Microsecond(e) => format_option!(f, e)?,
    4777
    0
                ScalarValue::Time64Nanosecond(e) => format_option!(f, e)?,
    4778
    0
                ScalarValue::IntervalYearMonth(e) => format_option!(f, e)?,
    4779
    0
                ScalarValue::IntervalMonthDayNano(e) => {
    4780
    0
                    format_option!(f, e.map(|v| format!("{v:?}")))?
    4781
                }
    4782
    0
                ScalarValue::IntervalDayTime(e) => {
    4783
    0
                    format_option!(f, e.map(|v| format!("{v:?}")))?;
    4784
                }
    4785
    0
                ScalarValue::DurationSecond(e) => format_option!(f, e)?,
    4786
    0
                ScalarValue::DurationMillisecond(e) => format_option!(f, e)?,
    4787
    0
                ScalarValue::DurationMicrosecond(e) => format_option!(f, e)?,
    4788
    0
                ScalarValue::DurationNanosecond(e) => format_option!(f, e)?,
    4789
    2
                ScalarValue::Struct(struct_arr) => {
    4790
                    // ScalarValue Struct should always have a single element
    4791
    2
                    assert_eq!(struct_arr.len(), 1);
    4792
    4793
    2
                    if struct_arr.null_count() == struct_arr.len() {
    4794
    0
                        write!(f, "NULL")?;
    4795
    0
                        return Ok(());
    4796
    2
                    }
    4797
    4798
    2
                    let columns = struct_arr.columns();
    4799
    2
                    let fields = struct_arr.fields();
    4800
    2
                    let nulls = struct_arr.nulls();
    4801
    4802
    2
                    write!(
    4803
    2
                        f,
    4804
    2
                        "{{{}}}",
    4805
    2
                        columns
    4806
    2
                            .iter()
    4807
    2
                            .zip(fields.iter())
    4808
    4
                            .map2(|(column, field)| {
    4809
    4
                                if nulls.is_some_and(|b| b0.is_null0(0)) {
    4810
    0
                                    format!("{}:NULL", field.name())
    4811
    4
                                } else if let DataType::Struct(_) = field.data_type() {
    4812
    0
                                    let sv = ScalarValue::Struct(Arc::new(
    4813
    0
                                        column.as_struct().to_owned(),
    4814
    0
                                    ));
    4815
    0
                                    format!("{}:{sv}", field.name())
    4816
                                } else {
    4817
    4
                                    let sv = array_value_to_string(column, 0).unwrap();
    4818
    4
                                    format!("{}:{sv}", field.name())
    4819
                                }
    4820
    4
                            })
    4821
    2
                            .collect::<Vec<_>>()
    4822
    2
                            .join(",")
    4823
    0
                    )?
    4824
                }
    4825
    0
                ScalarValue::Map(map_arr) => {
    4826
    0
                    if map_arr.null_count() == map_arr.len() {
    4827
    0
                        write!(f, "NULL")?;
    4828
    0
                        return Ok(());
    4829
    0
                    }
    4830
    4831
    0
                    write!(
    4832
    0
                        f,
    4833
    0
                        "[{}]",
    4834
    0
                        map_arr
    4835
    0
                            .iter()
    4836
    0
                            .map(|struct_array| {
    4837
    0
                                if let Some(arr) = struct_array {
    4838
    0
                                    let mut buffer = VecDeque::new();
    4839
    0
                                    for i in 0..arr.len() {
    4840
    0
                                        let key =
    4841
    0
                                            array_value_to_string(arr.column(0), i).unwrap();
    4842
    0
                                        let value =
    4843
    0
                                            array_value_to_string(arr.column(1), i).unwrap();
    4844
    0
                                        buffer.push_back(format!("{key}:{value}"));
    4845
    0
                                    }
    4846
    0
                                    format!(
    4847
    0
                                        "{{{}}}",
    4848
    0
                                        buffer
    4849
    0
                                            .into_iter()
    4850
    0
                                            .collect::<Vec<_>>()
    4851
    0
                                            .join(",")
    4852
    0
                                            .as_str()
    4853
                                    )
    4854
                                } else {
    4855
    0
                                    "NULL".to_string()
    4856
                                }
    4857
    0
                            })
    4858
    0
                            .collect::<Vec<_>>()
    4859
    0
                            .join(",")
    4860
    0
                    )?
    4861
                }
    4862
    0
                ScalarValue::Union(val, _fields, _mode) => match val {
    4863
    0
                    Some((id, val)) => write!(f, "{id}:{val}")?,
    4864
    0
                    None => write!(f, "NULL")?,
    4865
                },
    4866
    0
                ScalarValue::Dictionary(_k, v) => write!(f, "{v}")?,
    4867
    0
                ScalarValue::Null => write!(f, "NULL")?,
    4868
            };
    4869
    22
            Ok(())
    4870
    22
        }
    4871
    }
    4872
    4873
    0
    fn fmt_list(arr: &dyn Array, f: &mut fmt::Formatter) -> fmt::Result {
    4874
        // ScalarValue List, LargeList, FixedSizeList should always have a single element
    4875
    0
        assert_eq!(arr.len(), 1);
    4876
    0
        let options = FormatOptions::default().with_display_error(true);
    4877
    0
        let formatter = ArrayFormatter::try_new(arr, &options).unwrap();
    4878
    0
        let value_formatter = formatter.value(0);
    4879
    0
        write!(f, "{value_formatter}")
    4880
    0
    }
    4881
    4882
    /// writes a byte array to formatter. `[1, 2, 3]` ==> `"1,2,3"`
    4883
    0
    fn fmt_binary(data: &[u8], f: &mut fmt::Formatter) -> fmt::Result {
    4884
    0
        let mut iter = data.iter();
    4885
    0
        if let Some(b) = iter.next() {
    4886
    0
            write!(f, "{b}")?;
    4887
    0
        }
    4888
    0
        for b in iter {
    4889
    0
            write!(f, ",{b}")?;
    4890
        }
    4891
    0
        Ok(())
    4892
    0
    }
    4893
    4894
    impl fmt::Debug for ScalarValue {
    4895
    0
        fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
    4896
    0
            match self {
    4897
    0
                ScalarValue::Decimal32(_, _, _) => write!(f, "Decimal32({self})"),
    4898
    0
                ScalarValue::Decimal64(_, _, _) => write!(f, "Decimal64({self})"),
    4899
    0
                ScalarValue::Decimal128(_, _, _) => write!(f, "Decimal128({self})"),
    4900
    0
                ScalarValue::Decimal256(_, _, _) => write!(f, "Decimal256({self})"),
    4901
    0
                ScalarValue::Boolean(_) => write!(f, "Boolean({self})"),
    4902
    0
                ScalarValue::Float16(_) => write!(f, "Float16({self})"),
    4903
    0
                ScalarValue::Float32(_) => write!(f, "Float32({self})"),
    4904
    0
                ScalarValue::Float64(_) => write!(f, "Float64({self})"),
    4905
    0
                ScalarValue::Int8(_) => write!(f, "Int8({self})"),
    4906
    0
                ScalarValue::Int16(_) => write!(f, "Int16({self})"),
    4907
    0
                ScalarValue::Int32(_) => write!(f, "Int32({self})"),
    4908
    0
                ScalarValue::Int64(_) => write!(f, "Int64({self})"),
    4909
    0
                ScalarValue::UInt8(_) => write!(f, "UInt8({self})"),
    4910
    0
                ScalarValue::UInt16(_) => write!(f, "UInt16({self})"),
    4911
    0
                ScalarValue::UInt32(_) => write!(f, "UInt32({self})"),
    4912
    0
                ScalarValue::UInt64(_) => write!(f, "UInt64({self})"),
    4913
    0
                ScalarValue::TimestampSecond(_, tz_opt) => {
    4914
    0
                    write!(f, "TimestampSecond({self}, {tz_opt:?})")
    4915
                }
    4916
    0
                ScalarValue::TimestampMillisecond(_, tz_opt) => {
    4917
    0
                    write!(f, "TimestampMillisecond({self}, {tz_opt:?})")
    4918
                }
    4919
    0
                ScalarValue::TimestampMicrosecond(_, tz_opt) => {
    4920
    0
                    write!(f, "TimestampMicrosecond({self}, {tz_opt:?})")
    4921
                }
    4922
    0
                ScalarValue::TimestampNanosecond(_, tz_opt) => {
    4923
    0
                    write!(f, "TimestampNanosecond({self}, {tz_opt:?})")
    4924
                }
    4925
    0
                ScalarValue::Utf8(None) => write!(f, "Utf8({self})"),
    4926
    0
                ScalarValue::Utf8(Some(_)) => write!(f, "Utf8(\"{self}\")"),
    4927
    0
                ScalarValue::Utf8View(None) => write!(f, "Utf8View({self})"),
    4928
    0
                ScalarValue::Utf8View(Some(_)) => write!(f, "Utf8View(\"{self}\")"),
    4929
    0
                ScalarValue::LargeUtf8(None) => write!(f, "LargeUtf8({self})"),
    4930
    0
                ScalarValue::LargeUtf8(Some(_)) => write!(f, "LargeUtf8(\"{self}\")"),
    4931
    0
                ScalarValue::Binary(None) => write!(f, "Binary({self})"),
    4932
    0
                ScalarValue::Binary(Some(b)) => {
    4933
    0
                    write!(f, "Binary(\"")?;
    4934
    0
                    fmt_binary(b.as_slice(), f)?;
    4935
    0
                    write!(f, "\")")
    4936
                }
    4937
    0
                ScalarValue::BinaryView(None) => write!(f, "BinaryView({self})"),
    4938
    0
                ScalarValue::BinaryView(Some(b)) => {
    4939
    0
                    write!(f, "BinaryView(\"")?;
    4940
    0
                    fmt_binary(b.as_slice(), f)?;
    4941
    0
                    write!(f, "\")")
    4942
                }
    4943
    0
                ScalarValue::FixedSizeBinary(size, None) => {
    4944
    0
                    write!(f, "FixedSizeBinary({size}, {self})")
    4945
                }
    4946
    0
                ScalarValue::FixedSizeBinary(size, Some(b)) => {
    4947
    0
                    write!(f, "FixedSizeBinary({size}, \"")?;
    4948
    0
                    fmt_binary(b.as_slice(), f)?;
    4949
    0
                    write!(f, "\")")
    4950
                }
    4951
    0
                ScalarValue::LargeBinary(None) => write!(f, "LargeBinary({self})"),
    4952
    0
                ScalarValue::LargeBinary(Some(b)) => {
    4953
    0
                    write!(f, "LargeBinary(\"")?;
    4954
    0
                    fmt_binary(b.as_slice(), f)?;
    4955
    0
                    write!(f, "\")")
    4956
                }
    4957
    0
                ScalarValue::FixedSizeList(_) => write!(f, "FixedSizeList({self})"),
    4958
    0
                ScalarValue::List(_) => write!(f, "List({self})"),
    4959
    0
                ScalarValue::LargeList(_) => write!(f, "LargeList({self})"),
    4960
    0
                ScalarValue::Struct(struct_arr) => {
    4961
                    // ScalarValue Struct should always have a single element
    4962
    0
                    assert_eq!(struct_arr.len(), 1);
    4963
    4964
    0
                    let columns = struct_arr.columns();
    4965
    0
                    let fields = struct_arr.fields();
    4966
    4967
    0
                    write!(
    4968
    0
                        f,
    4969
    0
                        "Struct({{{}}})",
    4970
    0
                        columns
    4971
    0
                            .iter()
    4972
    0
                            .zip(fields.iter())
    4973
    0
                            .map(|(column, field)| {
    4974
    0
                                let sv = array_value_to_string(column, 0).unwrap();
    4975
    0
                                let name = field.name();
    4976
    0
                                format!("{name}:{sv}")
    4977
    0
                            })
    4978
    0
                            .collect::<Vec<_>>()
    4979
    0
                            .join(",")
    4980
                    )
    4981
                }
    4982
    0
                ScalarValue::Map(map_arr) => {
    4983
    0
                    write!(
    4984
    0
                        f,
    4985
    0
                        "Map([{}])",
    4986
    0
                        map_arr
    4987
    0
                            .iter()
    4988
    0
                            .map(|struct_array| {
    4989
    0
                                if let Some(arr) = struct_array {
    4990
    0
                                    let buffer: Vec<String> = (0..arr.len())
    4991
    0
                                        .map(|i| {
    4992
    0
                                            let key = array_value_to_string(arr.column(0), i)
    4993
    0
                                                .unwrap();
    4994
    0
                                            let value =
    4995
    0
                                                array_value_to_string(arr.column(1), i)
    4996
    0
                                                    .unwrap();
    4997
    0
                                            format!("{key:?}:{value:?}")
    4998
    0
                                        })
    4999
    0
                                        .collect();
    5000
    0
                                    format!("{{{}}}", buffer.join(","))
    5001
                                } else {
    5002
    0
                                    "NULL".to_string()
    5003
                                }
    5004
    0
                            })
    5005
    0
                            .collect::<Vec<_>>()
    5006
    0
                            .join(",")
    5007
                    )
    5008
                }
    5009
    0
                ScalarValue::Date32(_) => write!(f, "Date32(\"{self}\")"),
    5010
    0
                ScalarValue::Date64(_) => write!(f, "Date64(\"{self}\")"),
    5011
    0
                ScalarValue::Time32Second(_) => write!(f, "Time32Second(\"{self}\")"),
    5012
                ScalarValue::Time32Millisecond(_) => {
    5013
    0
                    write!(f, "Time32Millisecond(\"{self}\")")
    5014
                }
    5015
                ScalarValue::Time64Microsecond(_) => {
    5016
    0
                    write!(f, "Time64Microsecond(\"{self}\")")
    5017
                }
    5018
                ScalarValue::Time64Nanosecond(_) => {
    5019
    0
                    write!(f, "Time64Nanosecond(\"{self}\")")
    5020
                }
    5021
                ScalarValue::IntervalDayTime(_) => {
    5022
    0
                    write!(f, "IntervalDayTime(\"{self}\")")
    5023
                }
    5024
                ScalarValue::IntervalYearMonth(_) => {
    5025
    0
                    write!(f, "IntervalYearMonth(\"{self}\")")
    5026
                }
    5027
                ScalarValue::IntervalMonthDayNano(_) => {
    5028
    0
                    write!(f, "IntervalMonthDayNano(\"{self}\")")
    5029
                }
    5030
    0
                ScalarValue::DurationSecond(_) => write!(f, "DurationSecond(\"{self}\")"),
    5031
                ScalarValue::DurationMillisecond(_) => {
    5032
    0
                    write!(f, "DurationMillisecond(\"{self}\")")
    5033
                }
    5034
                ScalarValue::DurationMicrosecond(_) => {
    5035
    0
                    write!(f, "DurationMicrosecond(\"{self}\")")
    5036
                }
    5037
                ScalarValue::DurationNanosecond(_) => {
    5038
    0
                    write!(f, "DurationNanosecond(\"{self}\")")
    5039
                }
    5040
    0
                ScalarValue::Union(val, _fields, _mode) => match val {
    5041
    0
                    Some((id, val)) => write!(f, "Union {id}:{val}"),
    5042
    0
                    None => write!(f, "Union(NULL)"),
    5043
                },
    5044
    0
                ScalarValue::Dictionary(k, v) => write!(f, "Dictionary({k:?}, {v:?})"),
    5045
    0
                ScalarValue::Null => write!(f, "NULL"),
    5046
            }
    5047
    0
        }
    5048
    }
    5049
    5050
    /// Trait used to map a NativeType to a ScalarValue
    5051
    pub trait ScalarType<T: ArrowNativeType> {
    5052
        /// returns a scalar from an optional T
    5053
        fn scalar(r: Option<T>) -> ScalarValue;
    5054
    }
    5055
    5056
    impl ScalarType<f32> for Float32Type {
    5057
    0
        fn scalar(r: Option<f32>) -> ScalarValue {
    5058
    0
            ScalarValue::Float32(r)
    5059
    0
        }
    5060
    }
    5061
    5062
    impl ScalarType<i64> for TimestampSecondType {
    5063
    0
        fn scalar(r: Option<i64>) -> ScalarValue {
    5064
    0
            ScalarValue::TimestampSecond(r, None)
    5065
    0
        }
    5066
    }
    5067
    5068
    impl ScalarType<i64> for TimestampMillisecondType {
    5069
    0
        fn scalar(r: Option<i64>) -> ScalarValue {
    5070
    0
            ScalarValue::TimestampMillisecond(r, None)
    5071
    0
        }
    5072
    }
    5073
    5074
    impl ScalarType<i64> for TimestampMicrosecondType {
    5075
    0
        fn scalar(r: Option<i64>) -> ScalarValue {
    5076
    0
            ScalarValue::TimestampMicrosecond(r, None)
    5077
    0
        }
    5078
    }
    5079
    5080
    impl ScalarType<i64> for TimestampNanosecondType {
    5081
    0
        fn scalar(r: Option<i64>) -> ScalarValue {
    5082
    0
            ScalarValue::TimestampNanosecond(r, None)
    5083
    0
        }
    5084
    }
    5085
    5086
    impl ScalarType<i32> for Date32Type {
    5087
    0
        fn scalar(r: Option<i32>) -> ScalarValue {
    5088
    0
            ScalarValue::Date32(r)
    5089
    0
        }
    5090
    }
    5091
    5092
    #[cfg(test)]
    5093
    mod tests {
    5094
        use std::sync::Arc;
    5095
    5096
        use super::*;
    5097
        use crate::cast::{as_list_array, as_map_array, as_struct_array};
    5098
        use crate::test_util::batches_to_string;
    5099
        use arrow::array::{
    5100
            FixedSizeListBuilder, Int32Builder, LargeListBuilder, ListBuilder, MapBuilder,
    5101
            NullArray, NullBufferBuilder, OffsetSizeTrait, PrimitiveBuilder, RecordBatch,
    5102
            StringBuilder, StringDictionaryBuilder, StructBuilder, UnionBuilder,
    5103
        };
    5104
        use arrow::buffer::{Buffer, NullBuffer, OffsetBuffer};
    5105
        use arrow::compute::{is_null, kernels};
    5106
        use arrow::datatypes::{
    5107
            ArrowNumericType, Fields, Float64Type, TimeUnit, DECIMAL256_MAX_PRECISION,
    5108
        };
    5109
        use arrow::error::ArrowError;
    5110
        use arrow::util::pretty::pretty_format_columns;
    5111
        use chrono::NaiveDate;
    5112
        use insta::assert_snapshot;
    5113
        use rand::Rng;
    5114
    5115
        #[test]
    5116
        fn test_scalar_value_from_for_map() {
    5117
            let string_builder = StringBuilder::new();
    5118
            let int_builder = Int32Builder::with_capacity(4);
    5119
            let mut builder = MapBuilder::new(None, string_builder, int_builder);
    5120
            builder.keys().append_value("joe");
    5121
            builder.values().append_value(1);
    5122
            builder.append(true).unwrap();
    5123
    5124
            builder.keys().append_value("blogs");
    5125
            builder.values().append_value(2);
    5126
            builder.keys().append_value("foo");
    5127
            builder.values().append_value(4);
    5128
            builder.append(true).unwrap();
    5129
            builder.append(true).unwrap();
    5130
            builder.append(false).unwrap();
    5131
    5132
            let expected = builder.finish();
    5133
    5134
            let sv = ScalarValue::Map(Arc::new(expected.clone()));
    5135
            let map_arr = sv.to_array().unwrap();
    5136
            let actual = as_map_array(&map_arr).unwrap();
    5137
            assert_eq!(actual, &expected);
    5138
        }
    5139
    5140
        #[test]
    5141
        fn test_format_timestamp_type_for_error_and_bounds() {
    5142
            // format helper
    5143
            let ts_ns = format_timestamp_type_for_error(&DataType::Timestamp(
    5144
                TimeUnit::Nanosecond,
    5145
                None,
    5146
            ));
    5147
            assert_eq!(ts_ns, "Timestamp(ns)");
    5148
    5149
            let ts_us = format_timestamp_type_for_error(&DataType::Timestamp(
    5150
                TimeUnit::Microsecond,
    5151
                None,
    5152
            ));
    5153
            assert_eq!(ts_us, "Timestamp(us)");
    5154
    5155
            // ensure_timestamp_in_bounds: Date32 non-overflow
    5156
            let ok = ensure_timestamp_in_bounds(
    5157
                1000,
    5158
                NANOS_PER_DAY,
    5159
                &DataType::Date32,
    5160
                &DataType::Timestamp(TimeUnit::Nanosecond, None),
    5161
            );
    5162
            assert!(ok.is_ok());
    5163
    5164
            // Date32 overflow -- known large day value (9999-12-31 -> 2932896)
    5165
            let err = ensure_timestamp_in_bounds(
    5166
                2932896,
    5167
                NANOS_PER_DAY,
    5168
                &DataType::Date32,
    5169
                &DataType::Timestamp(TimeUnit::Nanosecond, None),
    5170
            );
    5171
            assert!(err.is_err());
    5172
            let msg = err.unwrap_err().to_string();
    5173
            assert!(msg.contains("Cannot cast Date32 value 2932896 to Timestamp(ns): converted value exceeds the representable i64 range"));
    5174
    5175
            // Date64 overflow for ns (millis * 1_000_000)
    5176
            let overflow_millis: i64 = (i64::MAX / NANOS_PER_MILLISECOND) + 1;
    5177
            let err2 = ensure_timestamp_in_bounds(
    5178
                overflow_millis,
    5179
                NANOS_PER_MILLISECOND,
    5180
                &DataType::Date64,
    5181
                &DataType::Timestamp(TimeUnit::Nanosecond, None),
    5182
            );
    5183
            assert!(err2.is_err());
    5184
        }
    5185
    5186
        #[test]
    5187
        fn test_scalar_value_from_for_struct() {
    5188
            let boolean = Arc::new(BooleanArray::from(vec![false]));
    5189
            let int = Arc::new(Int32Array::from(vec![42]));
    5190
    5191
            let expected = StructArray::from(vec![
    5192
                (
    5193
                    Arc::new(Field::new("b", DataType::Boolean, false)),
    5194
                    Arc::clone(&boolean) as ArrayRef,
    5195
                ),
    5196
                (
    5197
                    Arc::new(Field::new("c", DataType::Int32, false)),
    5198
                    Arc::clone(&int) as ArrayRef,
    5199
                ),
    5200
            ]);
    5201
    5202
            let sv = ScalarStructBuilder::new()
    5203
                .with_array(Field::new("b", DataType::Boolean, false), boolean)
    5204
                .with_array(Field::new("c", DataType::Int32, false), int)
    5205
                .build()
    5206
                .unwrap();
    5207
    5208
            let struct_arr = sv.to_array().unwrap();
    5209
            let actual = as_struct_array(&struct_arr).unwrap();
    5210
            assert_eq!(actual, &expected);
    5211
        }
    5212
    5213
        #[test]
    5214
        #[should_panic(
    5215
            expected = "InvalidArgumentError(\"Incorrect array length for StructArray field \\\"bool\\\", expected 1 got 4\")"
    5216
        )]
    5217
        fn test_scalar_value_from_for_struct_should_panic() {
    5218
            let _ = ScalarStructBuilder::new()
    5219
                .with_array(
    5220
                    Field::new("bool", DataType::Boolean, false),
    5221
                    Arc::new(BooleanArray::from(vec![false, true, false, false])),
    5222
                )
    5223
                .with_array(
    5224
                    Field::new("i32", DataType::Int32, false),
    5225
                    Arc::new(Int32Array::from(vec![42, 28, 19, 31])),
    5226
                )
    5227
                .build()
    5228
                .unwrap();
    5229
        }
    5230
    5231
        #[test]
    5232
        fn test_to_array_of_size_for_nested() {
    5233
            // Struct
    5234
            let boolean = Arc::new(BooleanArray::from(vec![false, false, true, true]));
    5235
            let int = Arc::new(Int32Array::from(vec![42, 28, 19, 31]));
    5236
    5237
            let struct_array = StructArray::from(vec![
    5238
                (
    5239
                    Arc::new(Field::new("b", DataType::Boolean, false)),
    5240
                    Arc::clone(&boolean) as ArrayRef,
    5241
                ),
    5242
                (
    5243
                    Arc::new(Field::new("c", DataType::Int32, false)),
    5244
                    Arc::clone(&int) as ArrayRef,
    5245
                ),
    5246
            ]);
    5247
            let sv = ScalarValue::Struct(Arc::new(struct_array));
    5248
            let actual_arr = sv.to_array_of_size(2).unwrap();
    5249
    5250
            let boolean = Arc::new(BooleanArray::from(vec![
    5251
                false, false, true, true, false, false, true, true,
    5252
            ]));
    5253
            let int = Arc::new(Int32Array::from(vec![42, 28, 19, 31, 42, 28, 19, 31]));
    5254
    5255
            let struct_array = StructArray::from(vec![
    5256
                (
    5257
                    Arc::new(Field::new("b", DataType::Boolean, false)),
    5258
                    Arc::clone(&boolean) as ArrayRef,
    5259
                ),
    5260
                (
    5261
                    Arc::new(Field::new("c", DataType::Int32, false)),
    5262
                    Arc::clone(&int) as ArrayRef,
    5263
                ),
    5264
            ]);
    5265
    5266
            let actual = as_struct_array(&actual_arr).unwrap();
    5267
            assert_eq!(actual, &struct_array);
    5268
    5269
            // List
    5270
            let arr = ListArray::from_iter_primitive::<Int32Type, _, _>(vec![Some(vec![
    5271
                Some(1),
    5272
                None,
    5273
                Some(2),
    5274
            ])]);
    5275
    5276
            let sv = ScalarValue::List(Arc::new(arr));
    5277
            let actual_arr = sv
    5278
                .to_array_of_size(2)
    5279
                .expect("Failed to convert to array of size");
    5280
            let actual_list_arr = actual_arr.as_list::<i32>();
    5281
    5282
            let arr = ListArray::from_iter_primitive::<Int32Type, _, _>(vec![
    5283
                Some(vec![Some(1), None, Some(2)]),
    5284
                Some(vec![Some(1), None, Some(2)]),
    5285
            ]);
    5286
    5287
            assert_eq!(&arr, actual_list_arr);
    5288
        }
    5289
    5290
        #[test]
    5291
        fn test_to_array_of_size_for_fsl() {
    5292
            let values = Int32Array::from_iter([Some(1), None, Some(2)]);
    5293
            let field = Arc::new(Field::new_list_field(DataType::Int32, true));
    5294
            let arr = FixedSizeListArray::new(Arc::clone(&field), 3, Arc::new(values), None);
    5295
            let sv = ScalarValue::FixedSizeList(Arc::new(arr));
    5296
            let actual_arr = sv
    5297
                .to_array_of_size(2)
    5298
                .expect("Failed to convert to array of size");
    5299
    5300
            let expected_values =
    5301
                Int32Array::from_iter([Some(1), None, Some(2), Some(1), None, Some(2)]);
    5302
            let expected_arr =
    5303
                FixedSizeListArray::new(field, 3, Arc::new(expected_values), None);
    5304
    5305
            assert_eq!(
    5306
                &expected_arr,
    5307
                as_fixed_size_list_array(actual_arr.as_ref()).unwrap()
    5308
            );
    5309
    5310
            let empty_array = sv
    5311
                .to_array_of_size(0)
    5312
                .expect("Failed to convert to empty array");
    5313
    5314
            assert_eq!(empty_array.len(), 0);
    5315
        }
    5316
    5317
        #[test]
    5318
        fn test_list_to_array_string() {
    5319
            let scalars = vec![
    5320
                ScalarValue::from("rust"),
    5321
                ScalarValue::from("arrow"),
    5322
                ScalarValue::from("data-fusion"),
    5323
            ];
    5324
    5325
            let result = ScalarValue::new_list_nullable(scalars.as_slice(), &DataType::Utf8);
    5326
    5327
            let expected = single_row_list_array(vec!["rust", "arrow", "data-fusion"]);
    5328
            assert_eq!(*result, expected);
    5329
        }
    5330
    5331
        fn single_row_list_array(items: Vec<&str>) -> ListArray {
    5332
            SingleRowListArrayBuilder::new(Arc::new(StringArray::from(items)))
    5333
                .build_list_array()
    5334
        }
    5335
    5336
        fn build_list<O: OffsetSizeTrait>(
    5337
            values: Vec<Option<Vec<Option<i64>>>>,
    5338
        ) -> Vec<ScalarValue> {
    5339
            values
    5340
                .into_iter()
    5341
                .map(|v| {
    5342
                    let arr = if v.is_some() {
    5343
                        Arc::new(
    5344
                            GenericListArray::<O>::from_iter_primitive::<Int64Type, _, _>(
    5345
                                vec![v],
    5346
                            ),
    5347
                        )
    5348
                    } else if O::IS_LARGE {
    5349
                        new_null_array(
    5350
                            &DataType::LargeList(Arc::new(Field::new_list_field(
    5351
                                DataType::Int64,
    5352
                                true,
    5353
                            ))),
    5354
                            1,
    5355
                        )
    5356
                    } else {
    5357
                        new_null_array(
    5358
                            &DataType::List(Arc::new(Field::new_list_field(
    5359
                                DataType::Int64,
    5360
                                true,
    5361
                            ))),
    5362
                            1,
    5363
                        )
    5364
                    };
    5365
    5366
                    if O::IS_LARGE {
    5367
                        ScalarValue::LargeList(arr.as_list::<i64>().to_owned().into())
    5368
                    } else {
    5369
                        ScalarValue::List(arr.as_list::<i32>().to_owned().into())
    5370
                    }
    5371
                })
    5372
                .collect()
    5373
        }
    5374
    5375
        #[test]
    5376
        fn test_iter_to_array_fixed_size_list() {
    5377
            let field = Arc::new(Field::new_list_field(DataType::Int32, true));
    5378
            let f1 = Arc::new(FixedSizeListArray::new(
    5379
                Arc::clone(&field),
    5380
                3,
    5381
                Arc::new(Int32Array::from(vec![1, 2, 3])),
    5382
                None,
    5383
            ));
    5384
            let f2 = Arc::new(FixedSizeListArray::new(
    5385
                Arc::clone(&field),
    5386
                3,
    5387
                Arc::new(Int32Array::from(vec![4, 5, 6])),
    5388
                None,
    5389
            ));
    5390
            let f_nulls = Arc::new(FixedSizeListArray::new_null(field, 1, 1));
    5391
    5392
            let scalars = vec![
    5393
                ScalarValue::FixedSizeList(Arc::clone(&f_nulls)),
    5394
                ScalarValue::FixedSizeList(f1),
    5395
                ScalarValue::FixedSizeList(f2),
    5396
                ScalarValue::FixedSizeList(f_nulls),
    5397
            ];
    5398
    5399
            let array = ScalarValue::iter_to_array(scalars).unwrap();
    5400
    5401
            let expected = FixedSizeListArray::from_iter_primitive::<Int32Type, _, _>(
    5402
                vec![
    5403
                    None,
    5404
                    Some(vec![Some(1), Some(2), Some(3)]),
    5405
                    Some(vec![Some(4), Some(5), Some(6)]),
    5406
                    None,
    5407
                ],
    5408
                3,
    5409
            );
    5410
            assert_eq!(array.as_ref(), &expected);
    5411
        }
    5412
    5413
        #[test]
    5414
        fn test_iter_to_array_struct() {
    5415
            let s1 = StructArray::from(vec![
    5416
                (
    5417
                    Arc::new(Field::new("A", DataType::Boolean, false)),
    5418
                    Arc::new(BooleanArray::from(vec![false])) as ArrayRef,
    5419
                ),
    5420
                (
    5421
                    Arc::new(Field::new("B", DataType::Int32, false)),
    5422
                    Arc::new(Int32Array::from(vec![42])) as ArrayRef,
    5423
                ),
    5424
            ]);
    5425
    5426
            let s2 = StructArray::from(vec![
    5427
                (
    5428
                    Arc::new(Field::new("A", DataType::Boolean, false)),
    5429
                    Arc::new(BooleanArray::from(vec![false])) as ArrayRef,
    5430
                ),
    5431
                (
    5432
                    Arc::new(Field::new("B", DataType::Int32, false)),
    5433
                    Arc::new(Int32Array::from(vec![42])) as ArrayRef,
    5434
                ),
    5435
            ]);
    5436
    5437
            let scalars = vec![
    5438
                ScalarValue::Struct(Arc::new(s1)),
    5439
                ScalarValue::Struct(Arc::new(s2)),
    5440
            ];
    5441
    5442
            let array = ScalarValue::iter_to_array(scalars).unwrap();
    5443
    5444
            let expected = StructArray::from(vec![
    5445
                (
    5446
                    Arc::new(Field::new("A", DataType::Boolean, false)),
    5447
                    Arc::new(BooleanArray::from(vec![false, false])) as ArrayRef,
    5448
                ),
    5449
                (
    5450
                    Arc::new(Field::new("B", DataType::Int32, false)),
    5451
                    Arc::new(Int32Array::from(vec![42, 42])) as ArrayRef,
    5452
                ),
    5453
            ]);
    5454
            assert_eq!(array.as_ref(), &expected);
    5455
        }
    5456
    5457
        #[test]
    5458
        fn test_iter_to_array_struct_with_nulls() {
    5459
            // non-null
    5460
            let s1 = StructArray::from((
    5461
                vec![
    5462
                    (
    5463
                        Arc::new(Field::new("A", DataType::Int32, false)),
    5464
                        Arc::new(Int32Array::from(vec![1])) as ArrayRef,
    5465
                    ),
    5466
                    (
    5467
                        Arc::new(Field::new("B", DataType::Int64, false)),
    5468
                        Arc::new(Int64Array::from(vec![2])) as ArrayRef,
    5469
                    ),
    5470
                ],
    5471
                // Present the null mask, 1 is non-null, 0 is null
    5472
                Buffer::from(&[1]),
    5473
            ));
    5474
    5475
            // null
    5476
            let s2 = StructArray::from((
    5477
                vec![
    5478
                    (
    5479
                        Arc::new(Field::new("A", DataType::Int32, false)),
    5480
                        Arc::new(Int32Array::from(vec![3])) as ArrayRef,
    5481
                    ),
    5482
                    (
    5483
                        Arc::new(Field::new("B", DataType::Int64, false)),
    5484
                        Arc::new(Int64Array::from(vec![4])) as ArrayRef,
    5485
                    ),
    5486
                ],
    5487
                Buffer::from(&[0]),
    5488
            ));
    5489
    5490
            let scalars = vec![
    5491
                ScalarValue::Struct(Arc::new(s1)),
    5492
                ScalarValue::Struct(Arc::new(s2)),
    5493
            ];
    5494
    5495
            let array = ScalarValue::iter_to_array(scalars).unwrap();
    5496
            let struct_array = array.as_struct();
    5497
            assert!(struct_array.is_valid(0));
    5498
            assert!(struct_array.is_null(1));
    5499
        }
    5500
    5501
        #[test]
    5502
        fn iter_to_array_primitive_test() {
    5503
            // List[[1,2,3]], List[null], List[[4,5]]
    5504
            let scalars = build_list::<i32>(vec![
    5505
                Some(vec![Some(1), Some(2), Some(3)]),
    5506
                None,
    5507
                Some(vec![Some(4), Some(5)]),
    5508
            ]);
    5509
    5510
            let array = ScalarValue::iter_to_array(scalars).unwrap();
    5511
            let list_array = as_list_array(&array).unwrap();
    5512
            // List[[1,2,3], null, [4,5]]
    5513
            let expected = ListArray::from_iter_primitive::<Int64Type, _, _>(vec![
    5514
                Some(vec![Some(1), Some(2), Some(3)]),
    5515
                None,
    5516
                Some(vec![Some(4), Some(5)]),
    5517
            ]);
    5518
            assert_eq!(list_array, &expected);
    5519
    5520
            let scalars = build_list::<i64>(vec![
    5521
                Some(vec![Some(1), Some(2), Some(3)]),
    5522
                None,
    5523
                Some(vec![Some(4), Some(5)]),
    5524
            ]);
    5525
    5526
            let array = ScalarValue::iter_to_array(scalars).unwrap();
    5527
            let list_array = as_large_list_array(&array).unwrap();
    5528
            let expected = LargeListArray::from_iter_primitive::<Int64Type, _, _>(vec![
    5529
                Some(vec![Some(1), Some(2), Some(3)]),
    5530
                None,
    5531
                Some(vec![Some(4), Some(5)]),
    5532
            ]);
    5533
            assert_eq!(list_array, &expected);
    5534
        }
    5535
    5536
        #[test]
    5537
        fn iter_to_array_string_test() {
    5538
            let arr1 = single_row_list_array(vec!["foo", "bar", "baz"]);
    5539
            let arr2 = single_row_list_array(vec!["rust", "world"]);
    5540
    5541
            let scalars = vec![
    5542
                ScalarValue::List(Arc::new(arr1)),
    5543
                ScalarValue::List(Arc::new(arr2)),
    5544
            ];
    5545
    5546
            let array = ScalarValue::iter_to_array(scalars).unwrap();
    5547
            let result = array.as_list::<i32>();
    5548
    5549
            // build expected array
    5550
            let string_builder = StringBuilder::with_capacity(5, 25);
    5551
            let mut list_of_string_builder = ListBuilder::new(string_builder);
    5552
    5553
            list_of_string_builder.values().append_value("foo");
    5554
            list_of_string_builder.values().append_value("bar");
    5555
            list_of_string_builder.values().append_value("baz");
    5556
            list_of_string_builder.append(true);
    5557
    5558
            list_of_string_builder.values().append_value("rust");
    5559
            list_of_string_builder.values().append_value("world");
    5560
            list_of_string_builder.append(true);
    5561
            let expected = list_of_string_builder.finish();
    5562
    5563
            assert_eq!(result, &expected);
    5564
        }
    5565
    5566
        #[test]
    5567
        fn test_list_scalar_eq_to_array() {
    5568
            let list_array: ArrayRef =
    5569
                Arc::new(ListArray::from_iter_primitive::<Int32Type, _, _>(vec![
    5570
                    Some(vec![Some(0), Some(1), Some(2)]),
    5571
                    None,
    5572
                    Some(vec![None, Some(5)]),
    5573
                ]));
    5574
    5575
            let fsl_array: ArrayRef =
    5576
                Arc::new(ListArray::from_iter_primitive::<Int32Type, _, _>(vec![
    5577
                    Some(vec![Some(0), Some(1), Some(2)]),
    5578
                    None,
    5579
                    Some(vec![Some(3), None, Some(5)]),
    5580
                ]));
    5581
    5582
            for arr in [list_array, fsl_array] {
    5583
                for i in 0..arr.len() {
    5584
                    let scalar =
    5585
                        ScalarValue::List(arr.slice(i, 1).as_list::<i32>().to_owned().into());
    5586
                    assert!(scalar.eq_array(&arr, i).unwrap());
    5587
                }
    5588
            }
    5589
        }
    5590
    5591
        #[test]
    5592
        fn test_eq_array_err_message() {
    5593
            assert_starts_with(
    5594
                ScalarValue::Utf8(Some("123".to_string()))
    5595
                    .eq_array(&(Arc::new(Int32Array::from(vec![123])) as ArrayRef), 0)
    5596
                    .unwrap_err()
    5597
                    .message(),
    5598
                "could not cast array of type Int32 to arrow_array::array::byte_array::GenericByteArray<arrow_array::types::GenericStringType<i32>>",
    5599
            );
    5600
        }
    5601
    5602
        #[test]
    5603
        fn scalar_add_trait_test() -> Result<()> {
    5604
            let float_value = ScalarValue::Float64(Some(123.));
    5605
            let float_value_2 = ScalarValue::Float64(Some(123.));
    5606
            assert_eq!(
    5607
                (float_value.add(&float_value_2))?,
    5608
                ScalarValue::Float64(Some(246.))
    5609
            );
    5610
            assert_eq!(
    5611
                (float_value.add(float_value_2))?,
    5612
                ScalarValue::Float64(Some(246.))
    5613
            );
    5614
            Ok(())
    5615
        }
    5616
    5617
        #[test]
    5618
        fn scalar_sub_trait_test() -> Result<()> {
    5619
            let float_value = ScalarValue::Float64(Some(123.));
    5620
            let float_value_2 = ScalarValue::Float64(Some(123.));
    5621
            assert_eq!(
    5622
                float_value.sub(&float_value_2)?,
    5623
                ScalarValue::Float64(Some(0.))
    5624
            );
    5625
            assert_eq!(
    5626
                float_value.sub(float_value_2)?,
    5627
                ScalarValue::Float64(Some(0.))
    5628
            );
    5629
            Ok(())
    5630
        }
    5631
    5632
        #[test]
    5633
        fn scalar_sub_trait_int32_test() -> Result<()> {
    5634
            let int_value = ScalarValue::Int32(Some(42));
    5635
            let int_value_2 = ScalarValue::Int32(Some(100));
    5636
            assert_eq!(int_value.sub(&int_value_2)?, ScalarValue::Int32(Some(-58)));
    5637
            assert_eq!(int_value_2.sub(int_value)?, ScalarValue::Int32(Some(58)));
    5638
            Ok(())
    5639
        }
    5640
    5641
        #[test]
    5642
        fn scalar_sub_trait_int32_overflow_test() {
    5643
            let int_value = ScalarValue::Int32(Some(i32::MAX));
    5644
            let int_value_2 = ScalarValue::Int32(Some(i32::MIN));
    5645
            let err = int_value
    5646
                .sub_checked(&int_value_2)
    5647
                .unwrap_err()
    5648
                .strip_backtrace();
    5649
            assert_eq!(
    5650
                err,
    5651
                "Arrow error: Arithmetic overflow: Overflow happened on: 2147483647 - -2147483648"
    5652
            )
    5653
        }
    5654
    5655
        #[test]
    5656
        fn scalar_sub_trait_int64_test() -> Result<()> {
    5657
            let int_value = ScalarValue::Int64(Some(42));
    5658
            let int_value_2 = ScalarValue::Int64(Some(100));
    5659
            assert_eq!(int_value.sub(&int_value_2)?, ScalarValue::Int64(Some(-58)));
    5660
            assert_eq!(int_value_2.sub(int_value)?, ScalarValue::Int64(Some(58)));
    5661
            Ok(())
    5662
        }
    5663
    5664
        #[test]
    5665
        fn scalar_sub_trait_int64_overflow_test() {
    5666
            let int_value = ScalarValue::Int64(Some(i64::MAX));
    5667
            let int_value_2 = ScalarValue::Int64(Some(i64::MIN));
    5668
            let err = int_value
    5669
                .sub_checked(&int_value_2)
    5670
                .unwrap_err()
    5671
                .strip_backtrace();
    5672
            assert_eq!(err, "Arrow error: Arithmetic overflow: Overflow happened on: 9223372036854775807 - -9223372036854775808")
    5673
        }
    5674
    5675
        #[test]
    5676
        fn scalar_add_overflow_test() -> Result<()> {
    5677
            check_scalar_add_overflow::<Int8Type>(
    5678
                ScalarValue::Int8(Some(i8::MAX)),
    5679
                ScalarValue::Int8(Some(i8::MAX)),
    5680
            );
    5681
            check_scalar_add_overflow::<UInt8Type>(
    5682
                ScalarValue::UInt8(Some(u8::MAX)),
    5683
                ScalarValue::UInt8(Some(u8::MAX)),
    5684
            );
    5685
            check_scalar_add_overflow::<Int16Type>(
    5686
                ScalarValue::Int16(Some(i16::MAX)),
    5687
                ScalarValue::Int16(Some(i16::MAX)),
    5688
            );
    5689
            check_scalar_add_overflow::<UInt16Type>(
    5690
                ScalarValue::UInt16(Some(u16::MAX)),
    5691
                ScalarValue::UInt16(Some(u16::MAX)),
    5692
            );
    5693
            check_scalar_add_overflow::<Int32Type>(
    5694
                ScalarValue::Int32(Some(i32::MAX)),
    5695
                ScalarValue::Int32(Some(i32::MAX)),
    5696
            );
    5697
            check_scalar_add_overflow::<UInt32Type>(
    5698
                ScalarValue::UInt32(Some(u32::MAX)),
    5699
                ScalarValue::UInt32(Some(u32::MAX)),
    5700
            );
    5701
            check_scalar_add_overflow::<Int64Type>(
    5702
                ScalarValue::Int64(Some(i64::MAX)),
    5703
                ScalarValue::Int64(Some(i64::MAX)),
    5704
            );
    5705
            check_scalar_add_overflow::<UInt64Type>(
    5706
                ScalarValue::UInt64(Some(u64::MAX)),
    5707
                ScalarValue::UInt64(Some(u64::MAX)),
    5708
            );
    5709
    5710
            Ok(())
    5711
        }
    5712
    5713
        // Verifies that ScalarValue has the same behavior with compute kernel when it overflows.
    5714
        fn check_scalar_add_overflow<T>(left: ScalarValue, right: ScalarValue)
    5715
        where
    5716
            T: ArrowNumericType,
    5717
        {
    5718
            let scalar_result = left.add_checked(&right);
    5719
    5720
            let left_array = left.to_array().expect("Failed to convert to array");
    5721
            let right_array = right.to_array().expect("Failed to convert to array");
    5722
            let arrow_left_array = left_array.as_primitive::<T>();
    5723
            let arrow_right_array = right_array.as_primitive::<T>();
    5724
            let arrow_result = add(arrow_left_array, arrow_right_array);
    5725
    5726
            assert_eq!(scalar_result.is_ok(), arrow_result.is_ok());
    5727
        }
    5728
    5729
        #[test]
    5730
        fn test_interval_add_timestamp() -> Result<()> {
    5731
            let interval = ScalarValue::IntervalMonthDayNano(Some(IntervalMonthDayNano {
    5732
                months: 1,
    5733
                days: 2,
    5734
                nanoseconds: 3,
    5735
            }));
    5736
            let timestamp = ScalarValue::TimestampNanosecond(Some(123), None);
    5737
            let result = interval.add(&timestamp)?;
    5738
            let expect = timestamp.add(&interval)?;
    5739
            assert_eq!(result, expect);
    5740
    5741
            let interval = ScalarValue::IntervalYearMonth(Some(123));
    5742
            let timestamp = ScalarValue::TimestampNanosecond(Some(123), None);
    5743
            let result = interval.add(&timestamp)?;
    5744
            let expect = timestamp.add(&interval)?;
    5745
            assert_eq!(result, expect);
    5746
    5747
            let interval = ScalarValue::IntervalDayTime(Some(IntervalDayTime {
    5748
                days: 1,
    5749
                milliseconds: 23,
    5750
            }));
    5751
            let timestamp = ScalarValue::TimestampNanosecond(Some(123), None);
    5752
            let result = interval.add(&timestamp)?;
    5753
            let expect = timestamp.add(&interval)?;
    5754
            assert_eq!(result, expect);
    5755
            Ok(())
    5756
        }
    5757
    5758
        #[test]
    5759
        fn test_try_cmp() {
    5760
            assert_eq!(
    5761
                ScalarValue::try_cmp(
    5762
                    &ScalarValue::Int32(Some(1)),
    5763
                    &ScalarValue::Int32(Some(2))
    5764
                )
    5765
                .unwrap(),
    5766
                Ordering::Less
    5767
            );
    5768
            assert_eq!(
    5769
                ScalarValue::try_cmp(&ScalarValue::Int32(None), &ScalarValue::Int32(Some(2)))
    5770
                    .unwrap(),
    5771
                Ordering::Less
    5772
            );
    5773
            assert_starts_with(
    5774
                ScalarValue::try_cmp(
    5775
                    &ScalarValue::Int32(Some(1)),
    5776
                    &ScalarValue::Int64(Some(2)),
    5777
                )
    5778
                .unwrap_err()
    5779
                .message(),
    5780
                "Uncomparable values: Int32(1), Int64(2)",
    5781
            );
    5782
        }
    5783
    5784
        #[test]
    5785
        fn scalar_decimal_test() -> Result<()> {
    5786
            let decimal_value = ScalarValue::Decimal128(Some(123), 10, 1);
    5787
            assert_eq!(DataType::Decimal128(10, 1), decimal_value.data_type());
    5788
            let try_into_value: i128 = decimal_value.clone().try_into().unwrap();
    5789
            assert_eq!(123_i128, try_into_value);
    5790
            assert!(!decimal_value.is_null());
    5791
            let neg_decimal_value = decimal_value.arithmetic_negate()?;
    5792
            match neg_decimal_value {
    5793
                ScalarValue::Decimal128(v, _, _) => {
    5794
                    assert_eq!(-123, v.unwrap());
    5795
                }
    5796
                _ => {
    5797
                    unreachable!();
    5798
                }
    5799
            }
    5800
    5801
            // decimal scalar to array
    5802
            let array = decimal_value
    5803
                .to_array()
    5804
                .expect("Failed to convert to array");
    5805
            let array = as_decimal128_array(&array)?;
    5806
            assert_eq!(1, array.len());
    5807
            assert_eq!(DataType::Decimal128(10, 1), array.data_type().clone());
    5808
            assert_eq!(123i128, array.value(0));
    5809
    5810
            // decimal scalar to array with size
    5811
            let array = decimal_value
    5812
                .to_array_of_size(10)
    5813
                .expect("Failed to convert to array of size");
    5814
            let array_decimal = as_decimal128_array(&array)?;
    5815
            assert_eq!(10, array.len());
    5816
            assert_eq!(DataType::Decimal128(10, 1), array.data_type().clone());
    5817
            assert_eq!(123i128, array_decimal.value(0));
    5818
            assert_eq!(123i128, array_decimal.value(9));
    5819
            // test eq array
    5820
            assert!(decimal_value
    5821
                .eq_array(&array, 1)
    5822
                .expect("Failed to compare arrays"));
    5823
            assert!(decimal_value
    5824
                .eq_array(&array, 5)
    5825
                .expect("Failed to compare arrays"));
    5826
            // test try from array
    5827
            assert_eq!(
    5828
                decimal_value,
    5829
                ScalarValue::try_from_array(&array, 5).unwrap()
    5830
            );
    5831
    5832
            assert_eq!(
    5833
                decimal_value,
    5834
                ScalarValue::try_new_decimal128(123, 10, 1).unwrap()
    5835
            );
    5836
    5837
            // test compare
    5838
            let left = ScalarValue::Decimal128(Some(123), 10, 2);
    5839
            let right = ScalarValue::Decimal128(Some(124), 10, 2);
    5840
            assert!(!left.eq(&right));
    5841
            let result = left < right;
    5842
            assert!(result);
    5843
            let result = left <= right;
    5844
            assert!(result);
    5845
            let right = ScalarValue::Decimal128(Some(124), 10, 3);
    5846
            // make sure that two decimals with diff datatype can't be compared.
    5847
            let result = left.partial_cmp(&right);
    5848
            assert_eq!(None, result);
    5849
    5850
            let decimal_vec = vec![
    5851
                ScalarValue::Decimal128(Some(1), 10, 2),
    5852
                ScalarValue::Decimal128(Some(2), 10, 2),
    5853
                ScalarValue::Decimal128(Some(3), 10, 2),
    5854
            ];
    5855
            // convert the vec to decimal array and check the result
    5856
            let array = ScalarValue::iter_to_array(decimal_vec).unwrap();
    5857
            assert_eq!(3, array.len());
    5858
            assert_eq!(DataType::Decimal128(10, 2), array.data_type().clone());
    5859
    5860
            let decimal_vec = vec![
    5861
                ScalarValue::Decimal128(Some(1), 10, 2),
    5862
                ScalarValue::Decimal128(Some(2), 10, 2),
    5863
                ScalarValue::Decimal128(Some(3), 10, 2),
    5864
                ScalarValue::Decimal128(None, 10, 2),
    5865
            ];
    5866
            let array = ScalarValue::iter_to_array(decimal_vec).unwrap();
    5867
            assert_eq!(4, array.len());
    5868
            assert_eq!(DataType::Decimal128(10, 2), array.data_type().clone());
    5869
    5870
            assert!(ScalarValue::try_new_decimal128(1, 10, 2)
    5871
                .unwrap()
    5872
                .eq_array(&array, 0)
    5873
                .expect("Failed to compare arrays"));
    5874
            assert!(ScalarValue::try_new_decimal128(2, 10, 2)
    5875
                .unwrap()
    5876
                .eq_array(&array, 1)
    5877
                .expect("Failed to compare arrays"));
    5878
            assert!(ScalarValue::try_new_decimal128(3, 10, 2)
    5879
                .unwrap()
    5880
                .eq_array(&array, 2)
    5881
                .expect("Failed to compare arrays"));
    5882
            assert_eq!(
    5883
                ScalarValue::Decimal128(None, 10, 2),
    5884
                ScalarValue::try_from_array(&array, 3).unwrap()
    5885
            );
    5886
    5887
            Ok(())
    5888
        }
    5889
    5890
        #[test]
    5891
        fn test_new_one_decimal128() {
    5892
            assert_eq!(
    5893
                ScalarValue::new_one(&DataType::Decimal128(5, 0)).unwrap(),
    5894
                ScalarValue::Decimal128(Some(1), 5, 0)
    5895
            );
    5896
            assert_eq!(
    5897
                ScalarValue::new_one(&DataType::Decimal128(5, 1)).unwrap(),
    5898
                ScalarValue::Decimal128(Some(10), 5, 1)
    5899
            );
    5900
            assert_eq!(
    5901
                ScalarValue::new_one(&DataType::Decimal128(5, 2)).unwrap(),
    5902
                ScalarValue::Decimal128(Some(100), 5, 2)
    5903
            );
    5904
            // More precision
    5905
            assert_eq!(
    5906
                ScalarValue::new_one(&DataType::Decimal128(7, 2)).unwrap(),
    5907
                ScalarValue::Decimal128(Some(100), 7, 2)
    5908
            );
    5909
            // No negative scale
    5910
            assert!(ScalarValue::new_one(&DataType::Decimal128(5, -1)).is_err());
    5911
            // Invalid combination
    5912
            assert!(ScalarValue::new_one(&DataType::Decimal128(0, 2)).is_err());
    5913
            assert!(ScalarValue::new_one(&DataType::Decimal128(5, 7)).is_err());
    5914
        }
    5915
    5916
        #[test]
    5917
        fn test_new_one_decimal256() {
    5918
            assert_eq!(
    5919
                ScalarValue::new_one(&DataType::Decimal256(5, 0)).unwrap(),
    5920
                ScalarValue::Decimal256(Some(1.into()), 5, 0)
    5921
            );
    5922
            assert_eq!(
    5923
                ScalarValue::new_one(&DataType::Decimal256(5, 1)).unwrap(),
    5924
                ScalarValue::Decimal256(Some(10.into()), 5, 1)
    5925
            );
    5926
            assert_eq!(
    5927
                ScalarValue::new_one(&DataType::Decimal256(5, 2)).unwrap(),
    5928
                ScalarValue::Decimal256(Some(100.into()), 5, 2)
    5929
            );
    5930
            // More precision
    5931
            assert_eq!(
    5932
                ScalarValue::new_one(&DataType::Decimal256(7, 2)).unwrap(),
    5933
                ScalarValue::Decimal256(Some(100.into()), 7, 2)
    5934
            );
    5935
            // No negative scale
    5936
            assert!(ScalarValue::new_one(&DataType::Decimal256(5, -1)).is_err());
    5937
            // Invalid combination
    5938
            assert!(ScalarValue::new_one(&DataType::Decimal256(0, 2)).is_err());
    5939
            assert!(ScalarValue::new_one(&DataType::Decimal256(5, 7)).is_err());
    5940
        }
    5941
    5942
        #[test]
    5943
        fn test_new_ten_decimal128() {
    5944
            assert_eq!(
    5945
                ScalarValue::new_ten(&DataType::Decimal128(5, 1)).unwrap(),
    5946
                ScalarValue::Decimal128(Some(100), 5, 1)
    5947
            );
    5948
            assert_eq!(
    5949
                ScalarValue::new_ten(&DataType::Decimal128(5, 2)).unwrap(),
    5950
                ScalarValue::Decimal128(Some(1000), 5, 2)
    5951
            );
    5952
            // More precision
    5953
            assert_eq!(
    5954
                ScalarValue::new_ten(&DataType::Decimal128(7, 2)).unwrap(),
    5955
                ScalarValue::Decimal128(Some(1000), 7, 2)
    5956
            );
    5957
            // No negative scale
    5958
            assert!(ScalarValue::new_ten(&DataType::Decimal128(5, -1)).is_err());
    5959
            // Invalid combination
    5960
            assert!(ScalarValue::new_ten(&DataType::Decimal128(0, 2)).is_err());
    5961
            assert!(ScalarValue::new_ten(&DataType::Decimal128(5, 7)).is_err());
    5962
        }
    5963
    5964
        #[test]
    5965
        fn test_new_ten_decimal256() {
    5966
            assert_eq!(
    5967
                ScalarValue::new_ten(&DataType::Decimal256(5, 1)).unwrap(),
    5968
                ScalarValue::Decimal256(Some(100.into()), 5, 1)
    5969
            );
    5970
            assert_eq!(
    5971
                ScalarValue::new_ten(&DataType::Decimal256(5, 2)).unwrap(),
    5972
                ScalarValue::Decimal256(Some(1000.into()), 5, 2)
    5973
            );
    5974
            // More precision
    5975
            assert_eq!(
    5976
                ScalarValue::new_ten(&DataType::Decimal256(7, 2)).unwrap(),
    5977
                ScalarValue::Decimal256(Some(1000.into()), 7, 2)
    5978
            );
    5979
            // No negative scale
    5980
            assert!(ScalarValue::new_ten(&DataType::Decimal256(5, -1)).is_err());
    5981
            // Invalid combination
    5982
            assert!(ScalarValue::new_ten(&DataType::Decimal256(0, 2)).is_err());
    5983
            assert!(ScalarValue::new_ten(&DataType::Decimal256(5, 7)).is_err());
    5984
        }
    5985
    5986
        #[test]
    5987
        fn test_new_negative_one_decimal128() {
    5988
            assert_eq!(
    5989
                ScalarValue::new_negative_one(&DataType::Decimal128(5, 0)).unwrap(),
    5990
                ScalarValue::Decimal128(Some(-1), 5, 0)
    5991
            );
    5992
            assert_eq!(
    5993
                ScalarValue::new_negative_one(&DataType::Decimal128(5, 2)).unwrap(),
    5994
                ScalarValue::Decimal128(Some(-100), 5, 2)
    5995
            );
    5996
        }
    5997
    5998
        #[test]
    5999
        fn test_list_partial_cmp() {
    6000
            let a =
    6001
                ScalarValue::List(Arc::new(
    6002
                    ListArray::from_iter_primitive::<Int64Type, _, _>(vec![Some(vec![
    6003
                        Some(1),
    6004
                        Some(2),
    6005
                        Some(3),
    6006
                    ])]),
    6007
                ));
    6008
            let b =
    6009
                ScalarValue::List(Arc::new(
    6010
                    ListArray::from_iter_primitive::<Int64Type, _, _>(vec![Some(vec![
    6011
                        Some(1),
    6012
                        Some(2),
    6013
                        Some(3),
    6014
                    ])]),
    6015
                ));
    6016
            assert_eq!(a.partial_cmp(&b), Some(Ordering::Equal));
    6017
    6018
            let a =
    6019
                ScalarValue::List(Arc::new(
    6020
                    ListArray::from_iter_primitive::<Int64Type, _, _>(vec![Some(vec![
    6021
                        Some(10),
    6022
                        Some(2),
    6023
                        Some(3),
    6024
                    ])]),
    6025
                ));
    6026
            let b =
    6027
                ScalarValue::List(Arc::new(
    6028
                    ListArray::from_iter_primitive::<Int64Type, _, _>(vec![Some(vec![
    6029
                        Some(1),
    6030
                        Some(2),
    6031
                        Some(30),
    6032
                    ])]),
    6033
                ));
    6034
            assert_eq!(a.partial_cmp(&b), Some(Ordering::Greater));
    6035
    6036
            let a =
    6037
                ScalarValue::List(Arc::new(
    6038
                    ListArray::from_iter_primitive::<Int64Type, _, _>(vec![Some(vec![
    6039
                        Some(10),
    6040
                        Some(2),
    6041
                        Some(3),
    6042
                    ])]),
    6043
                ));
    6044
            let b =
    6045
                ScalarValue::List(Arc::new(
    6046
                    ListArray::from_iter_primitive::<Int64Type, _, _>(vec![Some(vec![
    6047
                        Some(10),
    6048
                        Some(2),
    6049
                        Some(30),
    6050
                    ])]),
    6051
                ));
    6052
            assert_eq!(a.partial_cmp(&b), Some(Ordering::Less));
    6053
    6054
            let a =
    6055
                ScalarValue::List(Arc::new(
    6056
                    ListArray::from_iter_primitive::<Int64Type, _, _>(vec![Some(vec![
    6057
                        Some(1),
    6058
                        Some(2),
    6059
                        Some(3),
    6060
                    ])]),
    6061
                ));
    6062
            let b =
    6063
                ScalarValue::List(Arc::new(
    6064
                    ListArray::from_iter_primitive::<Int64Type, _, _>(vec![Some(vec![
    6065
                        Some(2),
    6066
                        Some(3),
    6067
                    ])]),
    6068
                ));
    6069
            assert_eq!(a.partial_cmp(&b), Some(Ordering::Less));
    6070
    6071
            let a =
    6072
                ScalarValue::List(Arc::new(
    6073
                    ListArray::from_iter_primitive::<Int64Type, _, _>(vec![Some(vec![
    6074
                        Some(2),
    6075
                        Some(3),
    6076
                        Some(4),
    6077
                    ])]),
    6078
                ));
    6079
            let b =
    6080
                ScalarValue::List(Arc::new(
    6081
                    ListArray::from_iter_primitive::<Int64Type, _, _>(vec![Some(vec![
    6082
                        Some(1),
    6083
                        Some(2),
    6084
                    ])]),
    6085
                ));
    6086
            assert_eq!(a.partial_cmp(&b), Some(Ordering::Greater));
    6087
    6088
            let a =
    6089
                ScalarValue::List(Arc::new(
    6090
                    ListArray::from_iter_primitive::<Int64Type, _, _>(vec![Some(vec![
    6091
                        Some(1),
    6092
                        Some(2),
    6093
                        Some(3),
    6094
                    ])]),
    6095
                ));
    6096
            let b =
    6097
                ScalarValue::List(Arc::new(
    6098
                    ListArray::from_iter_primitive::<Int64Type, _, _>(vec![Some(vec![
    6099
                        Some(1),
    6100
                        Some(2),
    6101
                    ])]),
    6102
                ));
    6103
            assert_eq!(a.partial_cmp(&b), Some(Ordering::Greater));
    6104
    6105
            let a =
    6106
                ScalarValue::List(Arc::new(
    6107
                    ListArray::from_iter_primitive::<Int64Type, _, _>(vec![Some(vec![
    6108
                        None,
    6109
                        Some(2),
    6110
                        Some(3),
    6111
                    ])]),
    6112
                ));
    6113
            let b =
    6114
                ScalarValue::List(Arc::new(
    6115
                    ListArray::from_iter_primitive::<Int64Type, _, _>(vec![Some(vec![
    6116
                        Some(1),
    6117
                        Some(2),
    6118
                        Some(3),
    6119
                    ])]),
    6120
                ));
    6121
            assert_eq!(a.partial_cmp(&b), Some(Ordering::Greater));
    6122
    6123
            let a = ScalarValue::LargeList(Arc::new(LargeListArray::from_iter_primitive::<
    6124
                Int64Type,
    6125
                _,
    6126
                _,
    6127
            >(vec![Some(vec![
    6128
                None,
    6129
                Some(2),
    6130
                Some(3),
    6131
            ])])));
    6132
            let b = ScalarValue::LargeList(Arc::new(LargeListArray::from_iter_primitive::<
    6133
                Int64Type,
    6134
                _,
    6135
                _,
    6136
            >(vec![Some(vec![
    6137
                Some(1),
    6138
                Some(2),
    6139
                Some(3),
    6140
            ])])));
    6141
            assert_eq!(a.partial_cmp(&b), Some(Ordering::Greater));
    6142
    6143
            let a = ScalarValue::FixedSizeList(Arc::new(
    6144
                FixedSizeListArray::from_iter_primitive::<Int64Type, _, _>(
    6145
                    vec![Some(vec![None, Some(2), Some(3)])],
    6146
                    3,
    6147
                ),
    6148
            ));
    6149
            let b = ScalarValue::FixedSizeList(Arc::new(
    6150
                FixedSizeListArray::from_iter_primitive::<Int64Type, _, _>(
    6151
                    vec![Some(vec![Some(1), Some(2), Some(3)])],
    6152
                    3,
    6153
                ),
    6154
            ));
    6155
            assert_eq!(a.partial_cmp(&b), Some(Ordering::Greater));
    6156
        }
    6157
    6158
        #[test]
    6159
        fn scalar_value_to_array_u64() -> Result<()> {
    6160
            let value = ScalarValue::UInt64(Some(13u64));
    6161
            let array = value.to_array().expect("Failed to convert to array");
    6162
            let array = as_uint64_array(&array)?;
    6163
            assert_eq!(array.len(), 1);
    6164
            assert!(!array.is_null(0));
    6165
            assert_eq!(array.value(0), 13);
    6166
    6167
            let value = ScalarValue::UInt64(None);
    6168
            let array = value.to_array().expect("Failed to convert to array");
    6169
            let array = as_uint64_array(&array)?;
    6170
            assert_eq!(array.len(), 1);
    6171
            assert!(array.is_null(0));
    6172
            Ok(())
    6173
        }
    6174
    6175
        #[test]
    6176
        fn scalar_value_to_array_u32() -> Result<()> {
    6177
            let value = ScalarValue::UInt32(Some(13u32));
    6178
            let array = value.to_array().expect("Failed to convert to array");
    6179
            let array = as_uint32_array(&array)?;
    6180
            assert_eq!(array.len(), 1);
    6181
            assert!(!array.is_null(0));
    6182
            assert_eq!(array.value(0), 13);
    6183
    6184
            let value = ScalarValue::UInt32(None);
    6185
            let array = value.to_array().expect("Failed to convert to array");
    6186
            let array = as_uint32_array(&array)?;
    6187
            assert_eq!(array.len(), 1);
    6188
            assert!(array.is_null(0));
    6189
            Ok(())
    6190
        }
    6191
    6192
        #[test]
    6193
        fn scalar_list_null_to_array() {
    6194
            let list_array = ScalarValue::new_list_nullable(&[], &DataType::UInt64);
    6195
    6196
            assert_eq!(list_array.len(), 1);
    6197
            assert_eq!(list_array.values().len(), 0);
    6198
        }
    6199
    6200
        #[test]
    6201
        fn scalar_large_list_null_to_array() {
    6202
            let list_array = ScalarValue::new_large_list(&[], &DataType::UInt64);
    6203
    6204
            assert_eq!(list_array.len(), 1);
    6205
            assert_eq!(list_array.values().len(), 0);
    6206
        }
    6207
    6208
        #[test]
    6209
        fn scalar_list_to_array() -> Result<()> {
    6210
            let values = vec![
    6211
                ScalarValue::UInt64(Some(100)),
    6212
                ScalarValue::UInt64(None),
    6213
                ScalarValue::UInt64(Some(101)),
    6214
            ];
    6215
            let list_array = ScalarValue::new_list_nullable(&values, &DataType::UInt64);
    6216
            assert_eq!(list_array.len(), 1);
    6217
            assert_eq!(list_array.values().len(), 3);
    6218
    6219
            let prim_array_ref = list_array.value(0);
    6220
            let prim_array = as_uint64_array(&prim_array_ref)?;
    6221
            assert_eq!(prim_array.len(), 3);
    6222
            assert_eq!(prim_array.value(0), 100);
    6223
            assert!(prim_array.is_null(1));
    6224
            assert_eq!(prim_array.value(2), 101);
    6225
            Ok(())
    6226
        }
    6227
    6228
        #[test]
    6229
        fn scalar_large_list_to_array() -> Result<()> {
    6230
            let values = vec![
    6231
                ScalarValue::UInt64(Some(100)),
    6232
                ScalarValue::UInt64(None),
    6233
                ScalarValue::UInt64(Some(101)),
    6234
            ];
    6235
            let list_array = ScalarValue::new_large_list(&values, &DataType::UInt64);
    6236
            assert_eq!(list_array.len(), 1);
    6237
            assert_eq!(list_array.values().len(), 3);
    6238
    6239
            let prim_array_ref = list_array.value(0);
    6240
            let prim_array = as_uint64_array(&prim_array_ref)?;
    6241
            assert_eq!(prim_array.len(), 3);
    6242
            assert_eq!(prim_array.value(0), 100);
    6243
            assert!(prim_array.is_null(1));
    6244
            assert_eq!(prim_array.value(2), 101);
    6245
            Ok(())
    6246
        }
    6247
    6248
        /// Creates array directly and via ScalarValue and ensures they are the same
    6249
        macro_rules! check_scalar_iter {
    6250
            ($SCALAR_T:ident, $ARRAYTYPE:ident, $INPUT:expr) => {{
    6251
                let scalars: Vec<_> =
    6252
                    $INPUT.iter().map(|v| ScalarValue::$SCALAR_T(*v)).collect();
    6253
    6254
                let array = ScalarValue::iter_to_array(scalars.into_iter()).unwrap();
    6255
    6256
                let expected: ArrayRef = Arc::new($ARRAYTYPE::from($INPUT));
    6257
    6258
                assert_eq!(&array, &expected);
    6259
            }};
    6260
        }
    6261
    6262
        /// Creates array directly and via ScalarValue and ensures they are the same
    6263
        /// but for variants that carry a timezone field.
    6264
        macro_rules! check_scalar_iter_tz {
    6265
            ($SCALAR_T:ident, $ARRAYTYPE:ident, $INPUT:expr) => {{
    6266
                let scalars: Vec<_> = $INPUT
    6267
                    .iter()
    6268
                    .map(|v| ScalarValue::$SCALAR_T(*v, None))
    6269
                    .collect();
    6270
    6271
                let array = ScalarValue::iter_to_array(scalars.into_iter()).unwrap();
    6272
    6273
                let expected: ArrayRef = Arc::new($ARRAYTYPE::from($INPUT));
    6274
    6275
                assert_eq!(&array, &expected);
    6276
            }};
    6277
        }
    6278
    6279
        /// Creates array directly and via ScalarValue and ensures they
    6280
        /// are the same, for string  arrays
    6281
        macro_rules! check_scalar_iter_string {
    6282
            ($SCALAR_T:ident, $ARRAYTYPE:ident, $INPUT:expr) => {{
    6283
                let scalars: Vec<_> = $INPUT
    6284
                    .iter()
    6285
                    .map(|v| ScalarValue::$SCALAR_T(v.map(|v| v.to_string())))
    6286
                    .collect();
    6287
    6288
                let array = ScalarValue::iter_to_array(scalars.into_iter()).unwrap();
    6289
    6290
                let expected: ArrayRef = Arc::new($ARRAYTYPE::from($INPUT));
    6291
    6292
                assert_eq!(&array, &expected);
    6293
            }};
    6294
        }
    6295
    6296
        /// Creates array directly and via ScalarValue and ensures they
    6297
        /// are the same, for binary arrays
    6298
        macro_rules! check_scalar_iter_binary {
    6299
            ($SCALAR_T:ident, $ARRAYTYPE:ident, $INPUT:expr) => {{
    6300
                let scalars: Vec<_> = $INPUT
    6301
                    .iter()
    6302
                    .map(|v| ScalarValue::$SCALAR_T(v.map(|v| v.to_vec())))
    6303
                    .collect();
    6304
    6305
                let array = ScalarValue::iter_to_array(scalars.into_iter()).unwrap();
    6306
    6307
                let expected: $ARRAYTYPE =
    6308
                    $INPUT.iter().map(|v| v.map(|v| v.to_vec())).collect();
    6309
    6310
                let expected: ArrayRef = Arc::new(expected);
    6311
    6312
                assert_eq!(&array, &expected);
    6313
            }};
    6314
        }
    6315
    6316
        #[test]
    6317
        // despite clippy claiming they are useless, the code doesn't compile otherwise.
    6318
        #[allow(clippy::useless_vec)]
    6319
        fn scalar_iter_to_array_boolean() {
    6320
            check_scalar_iter!(Boolean, BooleanArray, vec![Some(true), None, Some(false)]);
    6321
            check_scalar_iter!(Float32, Float32Array, vec![Some(1.9), None, Some(-2.1)]);
    6322
            check_scalar_iter!(Float64, Float64Array, vec![Some(1.9), None, Some(-2.1)]);
    6323
    6324
            check_scalar_iter!(Int8, Int8Array, vec![Some(1), None, Some(3)]);
    6325
            check_scalar_iter!(Int16, Int16Array, vec![Some(1), None, Some(3)]);
    6326
            check_scalar_iter!(Int32, Int32Array, vec![Some(1), None, Some(3)]);
    6327
            check_scalar_iter!(Int64, Int64Array, vec![Some(1), None, Some(3)]);
    6328
    6329
            check_scalar_iter!(UInt8, UInt8Array, vec![Some(1), None, Some(3)]);
    6330
            check_scalar_iter!(UInt16, UInt16Array, vec![Some(1), None, Some(3)]);
    6331
            check_scalar_iter!(UInt32, UInt32Array, vec![Some(1), None, Some(3)]);
    6332
            check_scalar_iter!(UInt64, UInt64Array, vec![Some(1), None, Some(3)]);
    6333
    6334
            check_scalar_iter_tz!(
    6335
                TimestampSecond,
    6336
                TimestampSecondArray,
    6337
                vec![Some(1), None, Some(3)]
    6338
            );
    6339
            check_scalar_iter_tz!(
    6340
                TimestampMillisecond,
    6341
                TimestampMillisecondArray,
    6342
                vec![Some(1), None, Some(3)]
    6343
            );
    6344
            check_scalar_iter_tz!(
    6345
                TimestampMicrosecond,
    6346
                TimestampMicrosecondArray,
    6347
                vec![Some(1), None, Some(3)]
    6348
            );
    6349
            check_scalar_iter_tz!(
    6350
                TimestampNanosecond,
    6351
                TimestampNanosecondArray,
    6352
                vec![Some(1), None, Some(3)]
    6353
            );
    6354
    6355
            check_scalar_iter_string!(
    6356
                Utf8,
    6357
                StringArray,
    6358
                vec![Some("foo"), None, Some("bar")]
    6359
            );
    6360
            check_scalar_iter_string!(
    6361
                LargeUtf8,
    6362
                LargeStringArray,
    6363
                vec![Some("foo"), None, Some("bar")]
    6364
            );
    6365
            check_scalar_iter_binary!(
    6366
                Binary,
    6367
                BinaryArray,
    6368
                vec![Some(b"foo"), None, Some(b"bar")]
    6369
            );
    6370
            check_scalar_iter_binary!(
    6371
                LargeBinary,
    6372
                LargeBinaryArray,
    6373
                vec![Some(b"foo"), None, Some(b"bar")]
    6374
            );
    6375
        }
    6376
    6377
        #[test]
    6378
        fn scalar_iter_to_array_empty() {
    6379
            let scalars = vec![] as Vec<ScalarValue>;
    6380
    6381
            let result = ScalarValue::iter_to_array(scalars).unwrap_err();
    6382
            assert!(
    6383
                result
    6384
                    .to_string()
    6385
                    .contains("Empty iterator passed to ScalarValue::iter_to_array"),
    6386
                "{}",
    6387
                result
    6388
            );
    6389
        }
    6390
    6391
        #[test]
    6392
        fn scalar_iter_to_dictionary() {
    6393
            fn make_val(v: Option<String>) -> ScalarValue {
    6394
                let key_type = DataType::Int32;
    6395
                let value = ScalarValue::Utf8(v);
    6396
                ScalarValue::Dictionary(Box::new(key_type), Box::new(value))
    6397
            }
    6398
    6399
            let scalars = [
    6400
                make_val(Some("Foo".into())),
    6401
                make_val(None),
    6402
                make_val(Some("Bar".into())),
    6403
            ];
    6404
    6405
            let array = ScalarValue::iter_to_array(scalars).unwrap();
    6406
            let array = as_dictionary_array::<Int32Type>(&array).unwrap();
    6407
            let values_array = as_string_array(array.values()).unwrap();
    6408
    6409
            let values = array
    6410
                .keys_iter()
    6411
                .map(|k| {
    6412
                    k.map(|k| {
    6413
                        assert!(values_array.is_valid(k));
    6414
                        values_array.value(k)
    6415
                    })
    6416
                })
    6417
                .collect::<Vec<_>>();
    6418
    6419
            let expected = vec![Some("Foo"), None, Some("Bar")];
    6420
            assert_eq!(values, expected);
    6421
        }
    6422
    6423
        #[test]
    6424
        fn scalar_iter_to_array_mismatched_types() {
    6425
            use ScalarValue::*;
    6426
            // If the scalar values are not all the correct type, error here
    6427
            let scalars = [Boolean(Some(true)), Int32(Some(5))];
    6428
    6429
            let result = ScalarValue::iter_to_array(scalars).unwrap_err();
    6430
            assert!(result.to_string().contains("Inconsistent types in ScalarValue::iter_to_array. Expected Boolean, got Int32(5)"),
    6431
                    "{}", result);
    6432
        }
    6433
    6434
        #[test]
    6435
        fn scalar_try_from_array_null() {
    6436
            let array = vec![Some(33), None].into_iter().collect::<Int64Array>();
    6437
            let array: ArrayRef = Arc::new(array);
    6438
    6439
            assert_eq!(
    6440
                ScalarValue::Int64(Some(33)),
    6441
                ScalarValue::try_from_array(&array, 0).unwrap()
    6442
            );
    6443
            assert_eq!(
    6444
                ScalarValue::Int64(None),
    6445
                ScalarValue::try_from_array(&array, 1).unwrap()
    6446
            );
    6447
        }
    6448
    6449
        #[test]
    6450
        fn scalar_try_from_array_list_array_null() {
    6451
            let list = ListArray::from_iter_primitive::<Int32Type, _, _>(vec![
    6452
                Some(vec![Some(1), Some(2)]),
    6453
                None,
    6454
            ]);
    6455
    6456
            let non_null_list_scalar = ScalarValue::try_from_array(&list, 0).unwrap();
    6457
            let null_list_scalar = ScalarValue::try_from_array(&list, 1).unwrap();
    6458
    6459
            let data_type =
    6460
                DataType::List(Arc::new(Field::new_list_field(DataType::Int32, true)));
    6461
    6462
            assert_eq!(non_null_list_scalar.data_type(), data_type);
    6463
            assert_eq!(null_list_scalar.data_type(), data_type);
    6464
        }
    6465
    6466
        #[test]
    6467
        fn scalar_try_from_list_datatypes() {
    6468
            let inner_field = Arc::new(Field::new_list_field(DataType::Int32, true));
    6469
    6470
            // Test for List
    6471
            let data_type = &DataType::List(Arc::clone(&inner_field));
    6472
            let scalar: ScalarValue = data_type.try_into().unwrap();
    6473
            let expected = ScalarValue::List(
    6474
                new_null_array(data_type, 1)
    6475
                    .as_list::<i32>()
    6476
                    .to_owned()
    6477
                    .into(),
    6478
            );
    6479
            assert_eq!(expected, scalar);
    6480
            assert!(expected.is_null());
    6481
    6482
            // Test for LargeList
    6483
            let data_type = &DataType::LargeList(Arc::clone(&inner_field));
    6484
            let scalar: ScalarValue = data_type.try_into().unwrap();
    6485
            let expected = ScalarValue::LargeList(
    6486
                new_null_array(data_type, 1)
    6487
                    .as_list::<i64>()
    6488
                    .to_owned()
    6489
                    .into(),
    6490
            );
    6491
            assert_eq!(expected, scalar);
    6492
            assert!(expected.is_null());
    6493
    6494
            // Test for FixedSizeList(5)
    6495
            let data_type = &DataType::FixedSizeList(Arc::clone(&inner_field), 5);
    6496
            let scalar: ScalarValue = data_type.try_into().unwrap();
    6497
            let expected = ScalarValue::FixedSizeList(
    6498
                new_null_array(data_type, 1)
    6499
                    .as_fixed_size_list()
    6500
                    .to_owned()
    6501
                    .into(),
    6502
            );
    6503
            assert_eq!(expected, scalar);
    6504
            assert!(expected.is_null());
    6505
        }
    6506
    6507
        #[test]
    6508
        fn scalar_try_from_list_of_list() {
    6509
            let data_type = DataType::List(Arc::new(Field::new_list_field(
    6510
                DataType::List(Arc::new(Field::new_list_field(DataType::Int32, true))),
    6511
                true,
    6512
            )));
    6513
            let data_type = &data_type;
    6514
            let scalar: ScalarValue = data_type.try_into().unwrap();
    6515
    6516
            let expected = ScalarValue::List(
    6517
                new_null_array(
    6518
                    &DataType::List(Arc::new(Field::new_list_field(
    6519
                        DataType::List(Arc::new(Field::new_list_field(
    6520
                            DataType::Int32,
    6521
                            true,
    6522
                        ))),
    6523
                        true,
    6524
                    ))),
    6525
                    1,
    6526
                )
    6527
                .as_list::<i32>()
    6528
                .to_owned()
    6529
                .into(),
    6530
            );
    6531
    6532
            assert_eq!(expected, scalar)
    6533
        }
    6534
    6535
        #[test]
    6536
        fn scalar_try_from_not_equal_list_nested_list() {
    6537
            let list_data_type =
    6538
                DataType::List(Arc::new(Field::new_list_field(DataType::Int32, true)));
    6539
            let data_type = &list_data_type;
    6540
            let list_scalar: ScalarValue = data_type.try_into().unwrap();
    6541
    6542
            let nested_list_data_type = DataType::List(Arc::new(Field::new_list_field(
    6543
                DataType::List(Arc::new(Field::new_list_field(DataType::Int32, true))),
    6544
                true,
    6545
            )));
    6546
            let data_type = &nested_list_data_type;
    6547
            let nested_list_scalar: ScalarValue = data_type.try_into().unwrap();
    6548
    6549
            assert_ne!(list_scalar, nested_list_scalar);
    6550
        }
    6551
    6552
        #[test]
    6553
        fn scalar_try_from_dict_datatype() {
    6554
            let data_type =
    6555
                DataType::Dictionary(Box::new(DataType::Int8), Box::new(DataType::Utf8));
    6556
            let data_type = &data_type;
    6557
            let expected = ScalarValue::Dictionary(
    6558
                Box::new(DataType::Int8),
    6559
                Box::new(ScalarValue::Utf8(None)),
    6560
            );
    6561
            assert_eq!(expected, data_type.try_into().unwrap())
    6562
        }
    6563
    6564
        #[test]
    6565
        fn size_of_scalar() {
    6566
            // Since ScalarValues are used in a non trivial number of places,
    6567
            // making it larger means significant more memory consumption
    6568
            // per distinct value.
    6569
            //
    6570
            // Thus this test ensures that no code change makes ScalarValue larger
    6571
            //
    6572
            // The alignment requirements differ across architectures and
    6573
            // thus the size of the enum appears to as well
    6574
    6575
            // The value may also change depending on rust version
    6576
            assert_eq!(size_of::<ScalarValue>(), 64);
    6577
        }
    6578
    6579
        #[test]
    6580
        fn memory_size() {
    6581
            let sv = ScalarValue::Binary(Some(Vec::with_capacity(10)));
    6582
            assert_eq!(sv.size(), size_of::<ScalarValue>() + 10,);
    6583
            let sv_size = sv.size();
    6584
    6585
            let mut v = Vec::with_capacity(10);
    6586
            // do NOT clone `sv` here because this may shrink the vector capacity
    6587
            v.push(sv);
    6588
            assert_eq!(v.capacity(), 10);
    6589
            assert_eq!(
    6590
                ScalarValue::size_of_vec(&v),
    6591
                size_of::<Vec<ScalarValue>>() + (9 * size_of::<ScalarValue>()) + sv_size,
    6592
            );
    6593
    6594
            let mut s = HashSet::with_capacity(0);
    6595
            // do NOT clone `sv` here because this may shrink the vector capacity
    6596
            s.insert(v.pop().unwrap());
    6597
            // hashsets may easily grow during insert, so capacity is dynamic
    6598
            let s_capacity = s.capacity();
    6599
            assert_eq!(
    6600
                ScalarValue::size_of_hashset(&s),
    6601
                size_of::<HashSet<ScalarValue>>()
    6602
                    + ((s_capacity - 1) * size_of::<ScalarValue>())
    6603
                    + sv_size,
    6604
            );
    6605
        }
    6606
    6607
        #[test]
    6608
        fn scalar_eq_array() {
    6609
            // Validate that eq_array has the same semantics as ScalarValue::eq
    6610
            macro_rules! make_typed_vec {
    6611
                ($INPUT:expr, $TYPE:ident) => {{
    6612
                    $INPUT
    6613
                        .iter()
    6614
                        .map(|v| v.map(|v| v as $TYPE))
    6615
                        .collect::<Vec<_>>()
    6616
                }};
    6617
            }
    6618
    6619
            let bool_vals = [Some(true), None, Some(false)];
    6620
            let f32_vals = [Some(-1.0), None, Some(1.0)];
    6621
            let f64_vals = make_typed_vec!(f32_vals, f64);
    6622
    6623
            let i8_vals = [Some(-1), None, Some(1)];
    6624
            let i16_vals = make_typed_vec!(i8_vals, i16);
    6625
            let i32_vals = make_typed_vec!(i8_vals, i32);
    6626
            let i64_vals = make_typed_vec!(i8_vals, i64);
    6627
    6628
            let u8_vals = [Some(0), None, Some(1)];
    6629
            let u16_vals = make_typed_vec!(u8_vals, u16);
    6630
            let u32_vals = make_typed_vec!(u8_vals, u32);
    6631
            let u64_vals = make_typed_vec!(u8_vals, u64);
    6632
    6633
            let str_vals = [Some("foo"), None, Some("bar")];
    6634
    6635
            let interval_dt_vals = [
    6636
                Some(IntervalDayTime::MINUS_ONE),
    6637
                None,
    6638
                Some(IntervalDayTime::ONE),
    6639
            ];
    6640
            let interval_mdn_vals = [
    6641
                Some(IntervalMonthDayNano::MINUS_ONE),
    6642
                None,
    6643
                Some(IntervalMonthDayNano::ONE),
    6644
            ];
    6645
    6646
            /// Test each value in `scalar` with the corresponding element
    6647
            /// at `array`. Assumes each element is unique (aka not equal
    6648
            /// with all other indexes)
    6649
            #[derive(Debug)]
    6650
            struct TestCase {
    6651
                array: ArrayRef,
    6652
                scalars: Vec<ScalarValue>,
    6653
            }
    6654
    6655
            /// Create a test case for casing the input to the specified array type
    6656
            macro_rules! make_test_case {
    6657
                ($INPUT:expr, $ARRAY_TY:ident, $SCALAR_TY:ident) => {{
    6658
                    TestCase {
    6659
                        array: Arc::new($INPUT.iter().collect::<$ARRAY_TY>()),
    6660
                        scalars: $INPUT.iter().map(|v| ScalarValue::$SCALAR_TY(*v)).collect(),
    6661
                    }
    6662
                }};
    6663
    6664
                ($INPUT:expr, $ARRAY_TY:ident, $SCALAR_TY:ident, $TZ:expr) => {{
    6665
                    let tz = $TZ;
    6666
                    TestCase {
    6667
                        array: Arc::new($INPUT.iter().collect::<$ARRAY_TY>()),
    6668
                        scalars: $INPUT
    6669
                            .iter()
    6670
                            .map(|v| ScalarValue::$SCALAR_TY(*v, tz.clone()))
    6671
                            .collect(),
    6672
                    }
    6673
                }};
    6674
            }
    6675
    6676
            macro_rules! make_str_test_case {
    6677
                ($INPUT:expr, $ARRAY_TY:ident, $SCALAR_TY:ident) => {{
    6678
                    TestCase {
    6679
                        array: Arc::new($INPUT.iter().cloned().collect::<$ARRAY_TY>()),
    6680
                        scalars: $INPUT
    6681
                            .iter()
    6682
                            .map(|v| ScalarValue::$SCALAR_TY(v.map(|v| v.to_string())))
    6683
                            .collect(),
    6684
                    }
    6685
                }};
    6686
            }
    6687
    6688
            macro_rules! make_binary_test_case {
    6689
                ($INPUT:expr, $ARRAY_TY:ident, $SCALAR_TY:ident) => {{
    6690
                    TestCase {
    6691
                        array: Arc::new($INPUT.iter().cloned().collect::<$ARRAY_TY>()),
    6692
                        scalars: $INPUT
    6693
                            .iter()
    6694
                            .map(|v| {
    6695
                                ScalarValue::$SCALAR_TY(v.map(|v| v.as_bytes().to_vec()))
    6696
                            })
    6697
                            .collect(),
    6698
                    }
    6699
                }};
    6700
            }
    6701
    6702
            /// create a test case for DictionaryArray<$INDEX_TY>
    6703
            macro_rules! make_str_dict_test_case {
    6704
                ($INPUT:expr, $INDEX_TY:ident) => {{
    6705
                    TestCase {
    6706
                        array: Arc::new(
    6707
                            $INPUT
    6708
                                .iter()
    6709
                                .cloned()
    6710
                                .collect::<DictionaryArray<$INDEX_TY>>(),
    6711
                        ),
    6712
                        scalars: $INPUT
    6713
                            .iter()
    6714
                            .map(|v| {
    6715
                                ScalarValue::Dictionary(
    6716
                                    Box::new($INDEX_TY::DATA_TYPE),
    6717
                                    Box::new(ScalarValue::Utf8(v.map(|v| v.to_string()))),
    6718
                                )
    6719
                            })
    6720
                            .collect(),
    6721
                    }
    6722
                }};
    6723
            }
    6724
    6725
            let cases = vec![
    6726
                make_test_case!(bool_vals, BooleanArray, Boolean),
    6727
                make_test_case!(f32_vals, Float32Array, Float32),
    6728
                make_test_case!(f64_vals, Float64Array, Float64),
    6729
                make_test_case!(i8_vals, Int8Array, Int8),
    6730
                make_test_case!(i16_vals, Int16Array, Int16),
    6731
                make_test_case!(i32_vals, Int32Array, Int32),
    6732
                make_test_case!(i64_vals, Int64Array, Int64),
    6733
                make_test_case!(u8_vals, UInt8Array, UInt8),
    6734
                make_test_case!(u16_vals, UInt16Array, UInt16),
    6735
                make_test_case!(u32_vals, UInt32Array, UInt32),
    6736
                make_test_case!(u64_vals, UInt64Array, UInt64),
    6737
                make_str_test_case!(str_vals, StringArray, Utf8),
    6738
                make_str_test_case!(str_vals, LargeStringArray, LargeUtf8),
    6739
                make_binary_test_case!(str_vals, BinaryArray, Binary),
    6740
                make_binary_test_case!(str_vals, LargeBinaryArray, LargeBinary),
    6741
                make_test_case!(i32_vals, Date32Array, Date32),
    6742
                make_test_case!(i64_vals, Date64Array, Date64),
    6743
                make_test_case!(i32_vals, Time32SecondArray, Time32Second),
    6744
                make_test_case!(i32_vals, Time32MillisecondArray, Time32Millisecond),
    6745
                make_test_case!(i64_vals, Time64MicrosecondArray, Time64Microsecond),
    6746
                make_test_case!(i64_vals, Time64NanosecondArray, Time64Nanosecond),
    6747
                make_test_case!(i64_vals, TimestampSecondArray, TimestampSecond, None),
    6748
                make_test_case!(
    6749
                    i64_vals,
    6750
                    TimestampSecondArray,
    6751
                    TimestampSecond,
    6752
                    Some("UTC".into())
    6753
                ),
    6754
                make_test_case!(
    6755
                    i64_vals,
    6756
                    TimestampMillisecondArray,
    6757
                    TimestampMillisecond,
    6758
                    None
    6759
                ),
    6760
                make_test_case!(
    6761
                    i64_vals,
    6762
                    TimestampMillisecondArray,
    6763
                    TimestampMillisecond,
    6764
                    Some("UTC".into())
    6765
                ),
    6766
                make_test_case!(
    6767
                    i64_vals,
    6768
                    TimestampMicrosecondArray,
    6769
                    TimestampMicrosecond,
    6770
                    None
    6771
                ),
    6772
                make_test_case!(
    6773
                    i64_vals,
    6774
                    TimestampMicrosecondArray,
    6775
                    TimestampMicrosecond,
    6776
                    Some("UTC".into())
    6777
                ),
    6778
                make_test_case!(
    6779
                    i64_vals,
    6780
                    TimestampNanosecondArray,
    6781
                    TimestampNanosecond,
    6782
                    None
    6783
                ),
    6784
                make_test_case!(
    6785
                    i64_vals,
    6786
                    TimestampNanosecondArray,
    6787
                    TimestampNanosecond,
    6788
                    Some("UTC".into())
    6789
                ),
    6790
                make_test_case!(i32_vals, IntervalYearMonthArray, IntervalYearMonth),
    6791
                make_test_case!(interval_dt_vals, IntervalDayTimeArray, IntervalDayTime),
    6792
                make_test_case!(
    6793
                    interval_mdn_vals,
    6794
                    IntervalMonthDayNanoArray,
    6795
                    IntervalMonthDayNano
    6796
                ),
    6797
                make_str_dict_test_case!(str_vals, Int8Type),
    6798
                make_str_dict_test_case!(str_vals, Int16Type),
    6799
                make_str_dict_test_case!(str_vals, Int32Type),
    6800
                make_str_dict_test_case!(str_vals, Int64Type),
    6801
                make_str_dict_test_case!(str_vals, UInt8Type),
    6802
                make_str_dict_test_case!(str_vals, UInt16Type),
    6803
                make_str_dict_test_case!(str_vals, UInt32Type),
    6804
                make_str_dict_test_case!(str_vals, UInt64Type),
    6805
            ];
    6806
    6807
            for case in cases {
    6808
                println!("**** Test Case *****");
    6809
                let TestCase { array, scalars } = case;
    6810
                println!("Input array type: {}", array.data_type());
    6811
                println!("Input scalars: {scalars:#?}");
    6812
                assert_eq!(array.len(), scalars.len());
    6813
    6814
                for (index, scalar) in scalars.into_iter().enumerate() {
    6815
                    assert!(
    6816
                        scalar
    6817
                            .eq_array(&array, index)
    6818
                            .expect("Failed to compare arrays"),
    6819
                        "Expected {scalar:?} to be equal to {array:?} at index {index}"
    6820
                    );
    6821
    6822
                    // test that all other elements are *not* equal
    6823
                    for other_index in 0..array.len() {
    6824
                        if index != other_index {
    6825
                            assert!(
    6826
                                !scalar.eq_array(&array, other_index).expect("Failed to compare arrays"),
    6827
                                "Expected {scalar:?} to be NOT equal to {array:?} at index {other_index}"
    6828
                            );
    6829
                        }
    6830
                    }
    6831
                }
    6832
            }
    6833
        }
    6834
    6835
        #[test]
    6836
        fn scalar_partial_ordering() {
    6837
            use ScalarValue::*;
    6838
    6839
            assert_eq!(
    6840
                Int64(Some(33)).partial_cmp(&Int64(Some(0))),
    6841
                Some(Ordering::Greater)
    6842
            );
    6843
            assert_eq!(
    6844
                Int64(Some(0)).partial_cmp(&Int64(Some(33))),
    6845
                Some(Ordering::Less)
    6846
            );
    6847
            assert_eq!(
    6848
                Int64(Some(33)).partial_cmp(&Int64(Some(33))),
    6849
                Some(Ordering::Equal)
    6850
            );
    6851
            // For different data type, `partial_cmp` returns None.
    6852
            assert_eq!(Int64(Some(33)).partial_cmp(&Int32(Some(33))), None);
    6853
            assert_eq!(Int32(Some(33)).partial_cmp(&Int64(Some(33))), None);
    6854
    6855
            assert_eq!(
    6856
                ScalarValue::from(vec![
    6857
                    ("A", ScalarValue::from(1.0)),
    6858
                    ("B", ScalarValue::from("Z")),
    6859
                ])
    6860
                .partial_cmp(&ScalarValue::from(vec![
    6861
                    ("A", ScalarValue::from(2.0)),
    6862
                    ("B", ScalarValue::from("A")),
    6863
                ])),
    6864
                Some(Ordering::Less)
    6865
            );
    6866
    6867
            // For different struct fields, `partial_cmp` returns None.
    6868
            assert_eq!(
    6869
                ScalarValue::from(vec![
    6870
                    ("A", ScalarValue::from(1.0)),
    6871
                    ("B", ScalarValue::from("Z")),
    6872
                ])
    6873
                .partial_cmp(&ScalarValue::from(vec![
    6874
                    ("a", ScalarValue::from(2.0)),
    6875
                    ("b", ScalarValue::from("A")),
    6876
                ])),
    6877
                None
    6878
            );
    6879
        }
    6880
    6881
        #[test]
    6882
        fn test_scalar_value_from_string() {
    6883
            let scalar = ScalarValue::from("foo");
    6884
            assert_eq!(scalar, ScalarValue::Utf8(Some("foo".to_string())));
    6885
            let scalar = ScalarValue::from("foo".to_string());
    6886
            assert_eq!(scalar, ScalarValue::Utf8(Some("foo".to_string())));
    6887
            let scalar = ScalarValue::from_str("foo").unwrap();
    6888
            assert_eq!(scalar, ScalarValue::Utf8(Some("foo".to_string())));
    6889
        }
    6890
    6891
        #[test]
    6892
        fn test_scalar_struct() {
    6893
            let field_a = Arc::new(Field::new("A", DataType::Int32, false));
    6894
            let field_b = Arc::new(Field::new("B", DataType::Boolean, false));
    6895
            let field_c = Arc::new(Field::new("C", DataType::Utf8, false));
    6896
    6897
            let field_e = Arc::new(Field::new("e", DataType::Int16, false));
    6898
            let field_f = Arc::new(Field::new("f", DataType::Int64, false));
    6899
            let field_d = Arc::new(Field::new(
    6900
                "D",
    6901
                DataType::Struct(vec![Arc::clone(&field_e), Arc::clone(&field_f)].into()),
    6902
                false,
    6903
            ));
    6904
    6905
            let struct_array = StructArray::from(vec![
    6906
                (
    6907
                    Arc::clone(&field_e),
    6908
                    Arc::new(Int16Array::from(vec![2])) as ArrayRef,
    6909
                ),
    6910
                (
    6911
                    Arc::clone(&field_f),
    6912
                    Arc::new(Int64Array::from(vec![3])) as ArrayRef,
    6913
                ),
    6914
            ]);
    6915
    6916
            let struct_array = StructArray::from(vec![
    6917
                (
    6918
                    Arc::clone(&field_a),
    6919
                    Arc::new(Int32Array::from(vec![23])) as ArrayRef,
    6920
                ),
    6921
                (
    6922
                    Arc::clone(&field_b),
    6923
                    Arc::new(BooleanArray::from(vec![false])) as ArrayRef,
    6924
                ),
    6925
                (
    6926
                    Arc::clone(&field_c),
    6927
                    Arc::new(StringArray::from(vec!["Hello"])) as ArrayRef,
    6928
                ),
    6929
                (Arc::clone(&field_d), Arc::new(struct_array) as ArrayRef),
    6930
            ]);
    6931
            let scalar = ScalarValue::Struct(Arc::new(struct_array));
    6932
    6933
            let array = scalar
    6934
                .to_array_of_size(2)
    6935
                .expect("Failed to convert to array of size");
    6936
    6937
            let expected = Arc::new(StructArray::from(vec![
    6938
                (
    6939
                    Arc::clone(&field_a),
    6940
                    Arc::new(Int32Array::from(vec![23, 23])) as ArrayRef,
    6941
                ),
    6942
                (
    6943
                    Arc::clone(&field_b),
    6944
                    Arc::new(BooleanArray::from(vec![false, false])) as ArrayRef,
    6945
                ),
    6946
                (
    6947
                    Arc::clone(&field_c),
    6948
                    Arc::new(StringArray::from(vec!["Hello", "Hello"])) as ArrayRef,
    6949
                ),
    6950
                (
    6951
                    Arc::clone(&field_d),
    6952
                    Arc::new(StructArray::from(vec![
    6953
                        (
    6954
                            Arc::clone(&field_e),
    6955
                            Arc::new(Int16Array::from(vec![2, 2])) as ArrayRef,
    6956
                        ),
    6957
                        (
    6958
                            Arc::clone(&field_f),
    6959
                            Arc::new(Int64Array::from(vec![3, 3])) as ArrayRef,
    6960
                        ),
    6961
                    ])) as ArrayRef,
    6962
                ),
    6963
            ])) as ArrayRef;
    6964
    6965
            assert_eq!(&array, &expected);
    6966
    6967
            // Construct from second element of ArrayRef
    6968
            let constructed = ScalarValue::try_from_array(&expected, 1).unwrap();
    6969
            assert_eq!(constructed, scalar);
    6970
    6971
            // None version
    6972
            let none_scalar = ScalarValue::try_from(array.data_type()).unwrap();
    6973
            assert!(none_scalar.is_null());
    6974
            assert_eq!(
    6975
                format!("{none_scalar:?}"),
    6976
                String::from("Struct({A:,B:,C:,D:})")
    6977
            );
    6978
    6979
            // Construct with convenience From<Vec<(&str, ScalarValue)>>
    6980
            let constructed = ScalarValue::from(vec![
    6981
                ("A", ScalarValue::from(23)),
    6982
                ("B", ScalarValue::from(false)),
    6983
                ("C", ScalarValue::from("Hello")),
    6984
                (
    6985
                    "D",
    6986
                    ScalarValue::from(vec![
    6987
                        ("e", ScalarValue::from(2i16)),
    6988
                        ("f", ScalarValue::from(3i64)),
    6989
                    ]),
    6990
                ),
    6991
            ]);
    6992
            assert_eq!(constructed, scalar);
    6993
    6994
            // Build Array from Vec of structs
    6995
            let scalars = vec![
    6996
                ScalarValue::from(vec![
    6997
                    ("A", ScalarValue::from(23)),
    6998
                    ("B", ScalarValue::from(false)),
    6999
                    ("C", ScalarValue::from("Hello")),
    7000
                    (
    7001
                        "D",
    7002
                        ScalarValue::from(vec![
    7003
                            ("e", ScalarValue::from(2i16)),
    7004
                            ("f", ScalarValue::from(3i64)),
    7005
                        ]),
    7006
                    ),
    7007
                ]),
    7008
                ScalarValue::from(vec![
    7009
                    ("A", ScalarValue::from(7)),
    7010
                    ("B", ScalarValue::from(true)),
    7011
                    ("C", ScalarValue::from("World")),
    7012
                    (
    7013
                        "D",
    7014
                        ScalarValue::from(vec![
    7015
                            ("e", ScalarValue::from(4i16)),
    7016
                            ("f", ScalarValue::from(5i64)),
    7017
                        ]),
    7018
                    ),
    7019
                ]),
    7020
                ScalarValue::from(vec![
    7021
                    ("A", ScalarValue::from(-1000)),
    7022
                    ("B", ScalarValue::from(true)),
    7023
                    ("C", ScalarValue::from("!!!!!")),
    7024
                    (
    7025
                        "D",
    7026
                        ScalarValue::from(vec![
    7027
                            ("e", ScalarValue::from(6i16)),
    7028
                            ("f", ScalarValue::from(7i64)),
    7029
                        ]),
    7030
                    ),
    7031
                ]),
    7032
            ];
    7033
            let array = ScalarValue::iter_to_array(scalars).unwrap();
    7034
    7035
            let expected = Arc::new(StructArray::from(vec![
    7036
                (
    7037
                    Arc::clone(&field_a),
    7038
                    Arc::new(Int32Array::from(vec![23, 7, -1000])) as ArrayRef,
    7039
                ),
    7040
                (
    7041
                    Arc::clone(&field_b),
    7042
                    Arc::new(BooleanArray::from(vec![false, true, true])) as ArrayRef,
    7043
                ),
    7044
                (
    7045
                    Arc::clone(&field_c),
    7046
                    Arc::new(StringArray::from(vec!["Hello", "World", "!!!!!"])) as ArrayRef,
    7047
                ),
    7048
                (
    7049
                    Arc::clone(&field_d),
    7050
                    Arc::new(StructArray::from(vec![
    7051
                        (
    7052
                            Arc::clone(&field_e),
    7053
                            Arc::new(Int16Array::from(vec![2, 4, 6])) as ArrayRef,
    7054
                        ),
    7055
                        (
    7056
                            Arc::clone(&field_f),
    7057
                            Arc::new(Int64Array::from(vec![3, 5, 7])) as ArrayRef,
    7058
                        ),
    7059
                    ])) as ArrayRef,
    7060
                ),
    7061
            ])) as ArrayRef;
    7062
    7063
            assert_eq!(&array, &expected);
    7064
        }
    7065
    7066
        #[test]
    7067
        fn round_trip() {
    7068
            // Each array type should be able to round tripped through a scalar
    7069
            let cases: Vec<ArrayRef> = vec![
    7070
                // int
    7071
                Arc::new(Int8Array::from(vec![Some(1), None, Some(3)])),
    7072
                Arc::new(Int16Array::from(vec![Some(1), None, Some(3)])),
    7073
                Arc::new(Int32Array::from(vec![Some(1), None, Some(3)])),
    7074
                Arc::new(Int64Array::from(vec![Some(1), None, Some(3)])),
    7075
                Arc::new(UInt8Array::from(vec![Some(1), None, Some(3)])),
    7076
                Arc::new(UInt16Array::from(vec![Some(1), None, Some(3)])),
    7077
                Arc::new(UInt32Array::from(vec![Some(1), None, Some(3)])),
    7078
                Arc::new(UInt64Array::from(vec![Some(1), None, Some(3)])),
    7079
                // bool
    7080
                Arc::new(BooleanArray::from(vec![Some(true), None, Some(false)])),
    7081
                // float
    7082
                Arc::new(Float32Array::from(vec![Some(1.0), None, Some(3.0)])),
    7083
                Arc::new(Float64Array::from(vec![Some(1.0), None, Some(3.0)])),
    7084
                // string array
    7085
                Arc::new(StringArray::from(vec![Some("foo"), None, Some("bar")])),
    7086
                Arc::new(LargeStringArray::from(vec![Some("foo"), None, Some("bar")])),
    7087
                Arc::new(StringViewArray::from(vec![Some("foo"), None, Some("bar")])),
    7088
                // string dictionary
    7089
                {
    7090
                    let mut builder = StringDictionaryBuilder::<Int32Type>::new();
    7091
                    builder.append("foo").unwrap();
    7092
                    builder.append_null();
    7093
                    builder.append("bar").unwrap();
    7094
                    Arc::new(builder.finish())
    7095
                },
    7096
                // binary array
    7097
                Arc::new(BinaryArray::from_iter(vec![
    7098
                    Some(b"foo"),
    7099
                    None,
    7100
                    Some(b"bar"),
    7101
                ])),
    7102
                Arc::new(LargeBinaryArray::from_iter(vec![
    7103
                    Some(b"foo"),
    7104
                    None,
    7105
                    Some(b"bar"),
    7106
                ])),
    7107
                Arc::new(BinaryViewArray::from_iter(vec![
    7108
                    Some(b"foo"),
    7109
                    None,
    7110
                    Some(b"bar"),
    7111
                ])),
    7112
                // timestamp
    7113
                Arc::new(TimestampSecondArray::from(vec![Some(1), None, Some(3)])),
    7114
                Arc::new(TimestampMillisecondArray::from(vec![
    7115
                    Some(1),
    7116
                    None,
    7117
                    Some(3),
    7118
                ])),
    7119
                Arc::new(TimestampMicrosecondArray::from(vec![
    7120
                    Some(1),
    7121
                    None,
    7122
                    Some(3),
    7123
                ])),
    7124
                Arc::new(TimestampNanosecondArray::from(vec![Some(1), None, Some(3)])),
    7125
                // timestamp with timezone
    7126
                Arc::new(
    7127
                    TimestampSecondArray::from(vec![Some(1), None, Some(3)])
    7128
                        .with_timezone_opt(Some("UTC")),
    7129
                ),
    7130
                Arc::new(
    7131
                    TimestampMillisecondArray::from(vec![Some(1), None, Some(3)])
    7132
                        .with_timezone_opt(Some("UTC")),
    7133
                ),
    7134
                Arc::new(
    7135
                    TimestampMicrosecondArray::from(vec![Some(1), None, Some(3)])
    7136
                        .with_timezone_opt(Some("UTC")),
    7137
                ),
    7138
                Arc::new(
    7139
                    TimestampNanosecondArray::from(vec![Some(1), None, Some(3)])
    7140
                        .with_timezone_opt(Some("UTC")),
    7141
                ),
    7142
                // date
    7143
                Arc::new(Date32Array::from(vec![Some(1), None, Some(3)])),
    7144
                Arc::new(Date64Array::from(vec![Some(1), None, Some(3)])),
    7145
                // time
    7146
                Arc::new(Time32SecondArray::from(vec![Some(1), None, Some(3)])),
    7147
                Arc::new(Time32MillisecondArray::from(vec![Some(1), None, Some(3)])),
    7148
                Arc::new(Time64MicrosecondArray::from(vec![Some(1), None, Some(3)])),
    7149
                Arc::new(Time64NanosecondArray::from(vec![Some(1), None, Some(3)])),
    7150
                // null array
    7151
                Arc::new(NullArray::new(3)),
    7152
                // dense union
    7153
                {
    7154
                    let mut builder = UnionBuilder::new_dense();
    7155
                    builder.append::<Int32Type>("a", 1).unwrap();
    7156
                    builder.append::<Float64Type>("b", 3.4).unwrap();
    7157
                    Arc::new(builder.build().unwrap())
    7158
                },
    7159
                // sparse union
    7160
                {
    7161
                    let mut builder = UnionBuilder::new_sparse();
    7162
                    builder.append::<Int32Type>("a", 1).unwrap();
    7163
                    builder.append::<Float64Type>("b", 3.4).unwrap();
    7164
                    Arc::new(builder.build().unwrap())
    7165
                },
    7166
                // list array
    7167
                {
    7168
                    let values_builder = StringBuilder::new();
    7169
                    let mut builder = ListBuilder::new(values_builder);
    7170
                    // [A, B]
    7171
                    builder.values().append_value("A");
    7172
                    builder.values().append_value("B");
    7173
                    builder.append(true);
    7174
                    // [ ] (empty list)
    7175
                    builder.append(true);
    7176
                    // Null
    7177
                    builder.values().append_value("?"); // irrelevant
    7178
                    builder.append(false);
    7179
                    Arc::new(builder.finish())
    7180
                },
    7181
                // large list array
    7182
                {
    7183
                    let values_builder = StringBuilder::new();
    7184
                    let mut builder = LargeListBuilder::new(values_builder);
    7185
                    // [A, B]
    7186
                    builder.values().append_value("A");
    7187
                    builder.values().append_value("B");
    7188
                    builder.append(true);
    7189
                    // [ ] (empty list)
    7190
                    builder.append(true);
    7191
                    // Null
    7192
                    builder.append(false);
    7193
                    Arc::new(builder.finish())
    7194
                },
    7195
                // fixed size list array
    7196
                {
    7197
                    let values_builder = Int32Builder::new();
    7198
                    let mut builder = FixedSizeListBuilder::new(values_builder, 3);
    7199
    7200
                    //  [[0, 1, 2], null, [3, null, 5]
    7201
                    builder.values().append_value(0);
    7202
                    builder.values().append_value(1);
    7203
                    builder.values().append_value(2);
    7204
                    builder.append(true);
    7205
                    builder.values().append_null();
    7206
                    builder.values().append_null();
    7207
                    builder.values().append_null();
    7208
                    builder.append(false);
    7209
                    builder.values().append_value(3);
    7210
                    builder.values().append_null();
    7211
                    builder.values().append_value(5);
    7212
                    builder.append(true);
    7213
                    Arc::new(builder.finish())
    7214
                },
    7215
                // map
    7216
                {
    7217
                    let string_builder = StringBuilder::new();
    7218
                    let int_builder = Int32Builder::with_capacity(4);
    7219
    7220
                    let mut builder = MapBuilder::new(None, string_builder, int_builder);
    7221
                    // {"joe": 1}
    7222
                    builder.keys().append_value("joe");
    7223
                    builder.values().append_value(1);
    7224
                    builder.append(true).unwrap();
    7225
                    // {}
    7226
                    builder.append(true).unwrap();
    7227
                    // null
    7228
                    builder.append(false).unwrap();
    7229
    7230
                    Arc::new(builder.finish())
    7231
                },
    7232
            ];
    7233
    7234
            for arr in cases {
    7235
                round_trip_through_scalar(arr);
    7236
            }
    7237
        }
    7238
    7239
        /// for each row in `arr`:
    7240
        /// 1. convert to a `ScalarValue`
    7241
        /// 2. Convert `ScalarValue` back to an `ArrayRef`
    7242
        /// 3. Compare the original array (sliced) and new array for equality
    7243
        fn round_trip_through_scalar(arr: ArrayRef) {
    7244
            for i in 0..arr.len() {
    7245
                // convert Scalar --> Array
    7246
                let scalar = ScalarValue::try_from_array(&arr, i).unwrap();
    7247
                let array = scalar.to_array_of_size(1).unwrap();
    7248
                assert_eq!(array.len(), 1);
    7249
                assert_eq!(array.data_type(), arr.data_type());
    7250
                assert_eq!(array.as_ref(), arr.slice(i, 1).as_ref());
    7251
            }
    7252
        }
    7253
    7254
        #[test]
    7255
        fn test_scalar_union_sparse() {
    7256
            let field_a = Arc::new(Field::new("A", DataType::Int32, true));
    7257
            let field_b = Arc::new(Field::new("B", DataType::Boolean, true));
    7258
            let field_c = Arc::new(Field::new("C", DataType::Utf8, true));
    7259
            let fields = UnionFields::from_iter([(0, field_a), (1, field_b), (2, field_c)]);
    7260
    7261
            let mut values_a = vec![None; 6];
    7262
            values_a[0] = Some(42);
    7263
            let mut values_b = vec![None; 6];
    7264
            values_b[1] = Some(true);
    7265
            let mut values_c = vec![None; 6];
    7266
            values_c[2] = Some("foo");
    7267
            let children: Vec<ArrayRef> = vec![
    7268
                Arc::new(Int32Array::from(values_a)),
    7269
                Arc::new(BooleanArray::from(values_b)),
    7270
                Arc::new(StringArray::from(values_c)),
    7271
            ];
    7272
    7273
            let type_ids = ScalarBuffer::from(vec![0, 1, 2, 0, 1, 2]);
    7274
            let array: ArrayRef = Arc::new(
    7275
                UnionArray::try_new(fields.clone(), type_ids, None, children)
    7276
                    .expect("UnionArray"),
    7277
            );
    7278
    7279
            let expected = [
    7280
                (0, ScalarValue::from(42)),
    7281
                (1, ScalarValue::from(true)),
    7282
                (2, ScalarValue::from("foo")),
    7283
                (0, ScalarValue::Int32(None)),
    7284
                (1, ScalarValue::Boolean(None)),
    7285
                (2, ScalarValue::Utf8(None)),
    7286
            ];
    7287
    7288
            for (i, (ti, value)) in expected.into_iter().enumerate() {
    7289
                let is_null = value.is_null();
    7290
                let value = Some((ti, Box::new(value)));
    7291
                let expected = ScalarValue::Union(value, fields.clone(), UnionMode::Sparse);
    7292
                let actual = ScalarValue::try_from_array(&array, i).expect("try_from_array");
    7293
    7294
                assert_eq!(
    7295
                    actual, expected,
    7296
                    "[{i}] {actual} was not equal to {expected}"
    7297
                );
    7298
    7299
                assert!(
    7300
                    expected.eq_array(&array, i).expect("eq_array"),
    7301
                    "[{i}] {expected}.eq_array was false"
    7302
                );
    7303
    7304
                if is_null {
    7305
                    assert!(actual.is_null(), "[{i}] {actual} was not null")
    7306
                }
    7307
            }
    7308
        }
    7309
    7310
        #[test]
    7311
        fn test_scalar_union_dense() {
    7312
            let field_a = Arc::new(Field::new("A", DataType::Int32, true));
    7313
            let field_b = Arc::new(Field::new("B", DataType::Boolean, true));
    7314
            let field_c = Arc::new(Field::new("C", DataType::Utf8, true));
    7315
            let fields = UnionFields::from_iter([(0, field_a), (1, field_b), (2, field_c)]);
    7316
            let children: Vec<ArrayRef> = vec![
    7317
                Arc::new(Int32Array::from(vec![Some(42), None])),
    7318
                Arc::new(BooleanArray::from(vec![Some(true), None])),
    7319
                Arc::new(StringArray::from(vec![Some("foo"), None])),
    7320
            ];
    7321
    7322
            let type_ids = ScalarBuffer::from(vec![0, 1, 2, 0, 1, 2]);
    7323
            let offsets = ScalarBuffer::from(vec![0, 0, 0, 1, 1, 1]);
    7324
            let array: ArrayRef = Arc::new(
    7325
                UnionArray::try_new(fields.clone(), type_ids, Some(offsets), children)
    7326
                    .expect("UnionArray"),
    7327
            );
    7328
    7329
            let expected = [
    7330
                (0, ScalarValue::from(42)),
    7331
                (1, ScalarValue::from(true)),
    7332
                (2, ScalarValue::from("foo")),
    7333
                (0, ScalarValue::Int32(None)),
    7334
                (1, ScalarValue::Boolean(None)),
    7335
                (2, ScalarValue::Utf8(None)),
    7336
            ];
    7337
    7338
            for (i, (ti, value)) in expected.into_iter().enumerate() {
    7339
                let is_null = value.is_null();
    7340
                let value = Some((ti, Box::new(value)));
    7341
                let expected = ScalarValue::Union(value, fields.clone(), UnionMode::Dense);
    7342
                let actual = ScalarValue::try_from_array(&array, i).expect("try_from_array");
    7343
    7344
                assert_eq!(
    7345
                    actual, expected,
    7346
                    "[{i}] {actual} was not equal to {expected}"
    7347
                );
    7348
    7349
                assert!(
    7350
                    expected.eq_array(&array, i).expect("eq_array"),
    7351
                    "[{i}] {expected}.eq_array was false"
    7352
                );
    7353
    7354
                if is_null {
    7355
                    assert!(actual.is_null(), "[{i}] {actual} was not null")
    7356
                }
    7357
            }
    7358
        }
    7359
    7360
        #[test]
    7361
        fn test_lists_in_struct() {
    7362
            let field_a = Arc::new(Field::new("A", DataType::Utf8, false));
    7363
            let field_primitive_list = Arc::new(Field::new(
    7364
                "primitive_list",
    7365
                DataType::List(Arc::new(Field::new_list_field(DataType::Int32, true))),
    7366
                false,
    7367
            ));
    7368
    7369
            // Define primitive list scalars
    7370
            let l0 =
    7371
                ScalarValue::List(Arc::new(
    7372
                    ListArray::from_iter_primitive::<Int32Type, _, _>(vec![Some(vec![
    7373
                        Some(1),
    7374
                        Some(2),
    7375
                        Some(3),
    7376
                    ])]),
    7377
                ));
    7378
            let l1 =
    7379
                ScalarValue::List(Arc::new(
    7380
                    ListArray::from_iter_primitive::<Int32Type, _, _>(vec![Some(vec![
    7381
                        Some(4),
    7382
                        Some(5),
    7383
                    ])]),
    7384
                ));
    7385
            let l2 = ScalarValue::List(Arc::new(ListArray::from_iter_primitive::<
    7386
                Int32Type,
    7387
                _,
    7388
                _,
    7389
            >(vec![Some(vec![Some(6)])])));
    7390
    7391
            // Define struct scalars
    7392
            let s0 = ScalarValue::from(vec![
    7393
                ("A", ScalarValue::from("First")),
    7394
                ("primitive_list", l0),
    7395
            ]);
    7396
    7397
            let s1 = ScalarValue::from(vec![
    7398
                ("A", ScalarValue::from("Second")),
    7399
                ("primitive_list", l1),
    7400
            ]);
    7401
    7402
            let s2 = ScalarValue::from(vec![
    7403
                ("A", ScalarValue::from("Third")),
    7404
                ("primitive_list", l2),
    7405
            ]);
    7406
    7407
            // iter_to_array for struct scalars
    7408
            let array =
    7409
                ScalarValue::iter_to_array(vec![s0.clone(), s1.clone(), s2.clone()]).unwrap();
    7410
    7411
            let array = as_struct_array(&array).unwrap();
    7412
            let expected = StructArray::from(vec![
    7413
                (
    7414
                    Arc::clone(&field_a),
    7415
                    Arc::new(StringArray::from(vec!["First", "Second", "Third"])) as ArrayRef,
    7416
                ),
    7417
                (
    7418
                    Arc::clone(&field_primitive_list),
    7419
                    Arc::new(ListArray::from_iter_primitive::<Int32Type, _, _>(vec![
    7420
                        Some(vec![Some(1), Some(2), Some(3)]),
    7421
                        Some(vec![Some(4), Some(5)]),
    7422
                        Some(vec![Some(6)]),
    7423
                    ])),
    7424
                ),
    7425
            ]);
    7426
    7427
            assert_eq!(array, &expected);
    7428
    7429
            // Define list-of-structs scalars
    7430
    7431
            let nl0_array = ScalarValue::iter_to_array(vec![s0, s1.clone()]).unwrap();
    7432
            let nl0 = SingleRowListArrayBuilder::new(nl0_array).build_list_scalar();
    7433
    7434
            let nl1_array = ScalarValue::iter_to_array(vec![s2]).unwrap();
    7435
            let nl1 = SingleRowListArrayBuilder::new(nl1_array).build_list_scalar();
    7436
    7437
            let nl2_array = ScalarValue::iter_to_array(vec![s1]).unwrap();
    7438
            let nl2 = SingleRowListArrayBuilder::new(nl2_array).build_list_scalar();
    7439
    7440
            // iter_to_array for list-of-struct
    7441
            let array = ScalarValue::iter_to_array(vec![nl0, nl1, nl2]).unwrap();
    7442
            let array = array.as_list::<i32>();
    7443
    7444
            // Construct expected array with array builders
    7445
            let field_a_builder = StringBuilder::with_capacity(4, 1024);
    7446
            let primitive_value_builder = Int32Array::builder(8);
    7447
            let field_primitive_list_builder = ListBuilder::new(primitive_value_builder);
    7448
    7449
            let element_builder = StructBuilder::new(
    7450
                vec![field_a, field_primitive_list],
    7451
                vec![
    7452
                    Box::new(field_a_builder),
    7453
                    Box::new(field_primitive_list_builder),
    7454
                ],
    7455
            );
    7456
    7457
            let mut list_builder = ListBuilder::new(element_builder);
    7458
    7459
            list_builder
    7460
                .values()
    7461
                .field_builder::<StringBuilder>(0)
    7462
                .unwrap()
    7463
                .append_value("First");
    7464
            list_builder
    7465
                .values()
    7466
                .field_builder::<ListBuilder<PrimitiveBuilder<Int32Type>>>(1)
    7467
                .unwrap()
    7468
                .values()
    7469
                .append_value(1);
    7470
            list_builder
    7471
                .values()
    7472
                .field_builder::<ListBuilder<PrimitiveBuilder<Int32Type>>>(1)
    7473
                .unwrap()
    7474
                .values()
    7475
                .append_value(2);
    7476
            list_builder
    7477
                .values()
    7478
                .field_builder::<ListBuilder<PrimitiveBuilder<Int32Type>>>(1)
    7479
                .unwrap()
    7480
                .values()
    7481
                .append_value(3);
    7482
            list_builder
    7483
                .values()
    7484
                .field_builder::<ListBuilder<PrimitiveBuilder<Int32Type>>>(1)
    7485
                .unwrap()
    7486
                .append(true);
    7487
            list_builder.values().append(true);
    7488
    7489
            list_builder
    7490
                .values()
    7491
                .field_builder::<StringBuilder>(0)
    7492
                .unwrap()
    7493
                .append_value("Second");
    7494
            list_builder
    7495
                .values()
    7496
                .field_builder::<ListBuilder<PrimitiveBuilder<Int32Type>>>(1)
    7497
                .unwrap()
    7498
                .values()
    7499
                .append_value(4);
    7500
            list_builder
    7501
                .values()
    7502
                .field_builder::<ListBuilder<PrimitiveBuilder<Int32Type>>>(1)
    7503
                .unwrap()
    7504
                .values()
    7505
                .append_value(5);
    7506
            list_builder
    7507
                .values()
    7508
                .field_builder::<ListBuilder<PrimitiveBuilder<Int32Type>>>(1)
    7509
                .unwrap()
    7510
                .append(true);
    7511
            list_builder.values().append(true);
    7512
            list_builder.append(true);
    7513
    7514
            list_builder
    7515
                .values()
    7516
                .field_builder::<StringBuilder>(0)
    7517
                .unwrap()
    7518
                .append_value("Third");
    7519
            list_builder
    7520
                .values()
    7521
                .field_builder::<ListBuilder<PrimitiveBuilder<Int32Type>>>(1)
    7522
                .unwrap()
    7523
                .values()
    7524
                .append_value(6);
    7525
            list_builder
    7526
                .values()
    7527
                .field_builder::<ListBuilder<PrimitiveBuilder<Int32Type>>>(1)
    7528
                .unwrap()
    7529
                .append(true);
    7530
            list_builder.values().append(true);
    7531
            list_builder.append(true);
    7532
    7533
            list_builder
    7534
                .values()
    7535
                .field_builder::<StringBuilder>(0)
    7536
                .unwrap()
    7537
                .append_value("Second");
    7538
            list_builder
    7539
                .values()
    7540
                .field_builder::<ListBuilder<PrimitiveBuilder<Int32Type>>>(1)
    7541
                .unwrap()
    7542
                .values()
    7543
                .append_value(4);
    7544
            list_builder
    7545
                .values()
    7546
                .field_builder::<ListBuilder<PrimitiveBuilder<Int32Type>>>(1)
    7547
                .unwrap()
    7548
                .values()
    7549
                .append_value(5);
    7550
            list_builder
    7551
                .values()
    7552
                .field_builder::<ListBuilder<PrimitiveBuilder<Int32Type>>>(1)
    7553
                .unwrap()
    7554
                .append(true);
    7555
            list_builder.values().append(true);
    7556
            list_builder.append(true);
    7557
    7558
            let expected = list_builder.finish();
    7559
    7560
            assert_eq!(array, &expected);
    7561
        }
    7562
    7563
        fn build_2d_list(data: Vec<Option<i32>>) -> ListArray {
    7564
            let a1 = ListArray::from_iter_primitive::<Int32Type, _, _>(vec![Some(data)]);
    7565
            ListArray::new(
    7566
                Arc::new(Field::new_list_field(
    7567
                    DataType::List(Arc::new(Field::new_list_field(DataType::Int32, true))),
    7568
                    true,
    7569
                )),
    7570
                OffsetBuffer::<i32>::from_lengths([1]),
    7571
                Arc::new(a1),
    7572
                None,
    7573
            )
    7574
        }
    7575
    7576
        #[test]
    7577
        fn test_nested_lists() {
    7578
            // Define inner list scalars
    7579
            let arr1 = build_2d_list(vec![Some(1), Some(2), Some(3)]);
    7580
            let arr2 = build_2d_list(vec![Some(4), Some(5)]);
    7581
            let arr3 = build_2d_list(vec![Some(6)]);
    7582
    7583
            let array = ScalarValue::iter_to_array(vec![
    7584
                ScalarValue::List(Arc::new(arr1)),
    7585
                ScalarValue::List(Arc::new(arr2)),
    7586
                ScalarValue::List(Arc::new(arr3)),
    7587
            ])
    7588
            .unwrap();
    7589
            let array = array.as_list::<i32>();
    7590
    7591
            // Construct expected array with array builders
    7592
            let inner_builder = Int32Array::builder(6);
    7593
            let middle_builder = ListBuilder::new(inner_builder);
    7594
            let mut outer_builder = ListBuilder::new(middle_builder);
    7595
    7596
            outer_builder.values().values().append_value(1);
    7597
            outer_builder.values().values().append_value(2);
    7598
            outer_builder.values().values().append_value(3);
    7599
            outer_builder.values().append(true);
    7600
            outer_builder.append(true);
    7601
    7602
            outer_builder.values().values().append_value(4);
    7603
            outer_builder.values().values().append_value(5);
    7604
            outer_builder.values().append(true);
    7605
            outer_builder.append(true);
    7606
    7607
            outer_builder.values().values().append_value(6);
    7608
            outer_builder.values().append(true);
    7609
            outer_builder.append(true);
    7610
    7611
            let expected = outer_builder.finish();
    7612
    7613
            assert_eq!(array, &expected);
    7614
        }
    7615
    7616
        #[test]
    7617
        fn scalar_timestamp_ns_utc_timezone() {
    7618
            let scalar = ScalarValue::TimestampNanosecond(
    7619
                Some(1599566400000000000),
    7620
                Some("UTC".into()),
    7621
            );
    7622
    7623
            assert_eq!(
    7624
                scalar.data_type(),
    7625
                DataType::Timestamp(TimeUnit::Nanosecond, Some("UTC".into()))
    7626
            );
    7627
    7628
            let array = scalar.to_array().expect("Failed to convert to array");
    7629
            assert_eq!(array.len(), 1);
    7630
            assert_eq!(
    7631
                array.data_type(),
    7632
                &DataType::Timestamp(TimeUnit::Nanosecond, Some("UTC".into()))
    7633
            );
    7634
    7635
            let new_scalar = ScalarValue::try_from_array(&array, 0).unwrap();
    7636
            assert_eq!(
    7637
                new_scalar.data_type(),
    7638
                DataType::Timestamp(TimeUnit::Nanosecond, Some("UTC".into()))
    7639
            );
    7640
        }
    7641
    7642
        #[test]
    7643
        fn cast_round_trip() {
    7644
            check_scalar_cast(ScalarValue::Int8(Some(5)), DataType::Int16);
    7645
            check_scalar_cast(ScalarValue::Int8(None), DataType::Int16);
    7646
    7647
            check_scalar_cast(ScalarValue::Float64(Some(5.5)), DataType::Int16);
    7648
    7649
            check_scalar_cast(ScalarValue::Float64(None), DataType::Int16);
    7650
    7651
            check_scalar_cast(
    7652
                ScalarValue::from("foo"),
    7653
                DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)),
    7654
            );
    7655
    7656
            check_scalar_cast(
    7657
                ScalarValue::Utf8(None),
    7658
                DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)),
    7659
            );
    7660
    7661
            check_scalar_cast(ScalarValue::Utf8(None), DataType::Utf8View);
    7662
            check_scalar_cast(ScalarValue::from("foo"), DataType::Utf8View);
    7663
            check_scalar_cast(
    7664
                ScalarValue::from("larger than 12 bytes string"),
    7665
                DataType::Utf8View,
    7666
            );
    7667
            check_scalar_cast(
    7668
                {
    7669
                    let element_field =
    7670
                        Arc::new(Field::new("element", DataType::Int32, true));
    7671
    7672
                    let mut builder =
    7673
                        ListBuilder::new(Int32Builder::new()).with_field(element_field);
    7674
                    builder.append_value([Some(1)]);
    7675
                    builder.append(true);
    7676
    7677
                    ScalarValue::List(Arc::new(builder.finish()))
    7678
                },
    7679
                DataType::List(Arc::new(Field::new("element", DataType::Int64, true))),
    7680
            );
    7681
            check_scalar_cast(
    7682
                {
    7683
                    let element_field =
    7684
                        Arc::new(Field::new("element", DataType::Int32, true));
    7685
    7686
                    let mut builder = FixedSizeListBuilder::new(Int32Builder::new(), 1)
    7687
                        .with_field(element_field);
    7688
                    builder.values().append_value(1);
    7689
                    builder.append(true);
    7690
    7691
                    ScalarValue::FixedSizeList(Arc::new(builder.finish()))
    7692
                },
    7693
                DataType::FixedSizeList(
    7694
                    Arc::new(Field::new("element", DataType::Int64, true)),
    7695
                    1,
    7696
                ),
    7697
            );
    7698
            check_scalar_cast(
    7699
                {
    7700
                    let element_field =
    7701
                        Arc::new(Field::new("element", DataType::Int32, true));
    7702
    7703
                    let mut builder =
    7704
                        LargeListBuilder::new(Int32Builder::new()).with_field(element_field);
    7705
                    builder.append_value([Some(1)]);
    7706
                    builder.append(true);
    7707
    7708
                    ScalarValue::LargeList(Arc::new(builder.finish()))
    7709
                },
    7710
                DataType::LargeList(Arc::new(Field::new("element", DataType::Int64, true))),
    7711
            );
    7712
        }
    7713
    7714
        // mimics how casting work on scalar values by `casting` `scalar` to `desired_type`
    7715
        fn check_scalar_cast(scalar: ScalarValue, desired_type: DataType) {
    7716
            // convert from scalar --> Array to call cast
    7717
            let scalar_array = scalar.to_array().expect("Failed to convert to array");
    7718
            // cast the actual value
    7719
            let cast_array = kernels::cast::cast(&scalar_array, &desired_type).unwrap();
    7720
    7721
            // turn it back to a scalar
    7722
            let cast_scalar = ScalarValue::try_from_array(&cast_array, 0).unwrap();
    7723
            assert_eq!(cast_scalar.data_type(), desired_type);
    7724
    7725
            // Some time later the "cast" scalar is turned back into an array:
    7726
            let array = cast_scalar
    7727
                .to_array_of_size(10)
    7728
                .expect("Failed to convert to array of size");
    7729
    7730
            // The datatype should be "Dictionary" but is actually Utf8!!!
    7731
            assert_eq!(array.data_type(), &desired_type)
    7732
        }
    7733
    7734
        #[test]
    7735
        fn test_scalar_negative() -> Result<()> {
    7736
            // positive test
    7737
            let value = ScalarValue::Int32(Some(12));
    7738
            assert_eq!(ScalarValue::Int32(Some(-12)), value.arithmetic_negate()?);
    7739
            let value = ScalarValue::Int32(None);
    7740
            assert_eq!(ScalarValue::Int32(None), value.arithmetic_negate()?);
    7741
    7742
            // negative test
    7743
            let value = ScalarValue::UInt8(Some(12));
    7744
            assert!(value.arithmetic_negate().is_err());
    7745
            let value = ScalarValue::Boolean(None);
    7746
            assert!(value.arithmetic_negate().is_err());
    7747
            Ok(())
    7748
        }
    7749
    7750
        #[test]
    7751
        #[allow(arithmetic_overflow)] // we want to test them
    7752
        fn test_scalar_negative_overflows() -> Result<()> {
    7753
            macro_rules! test_overflow_on_value {
    7754
                ($($val:expr),* $(,)?) => {$(
    7755
                    {
    7756
                        let value: ScalarValue = $val;
    7757
                        let err = value.arithmetic_negate().expect_err("Should receive overflow error on negating {value:?}");
    7758
                        let root_err = err.find_root();
    7759
                        match  root_err{
    7760
                            DataFusionError::ArrowError(err, _) if matches!(err.as_ref(), ArrowError::ArithmeticOverflow(_)) => {}
    7761
                            _ => return Err(err),
    7762
                        };
    7763
                    }
    7764
                )*};
    7765
            }
    7766
            test_overflow_on_value!(
    7767
                // the integers
    7768
                i8::MIN.into(),
    7769
                i16::MIN.into(),
    7770
                i32::MIN.into(),
    7771
                i64::MIN.into(),
    7772
                // for decimals, only value needs to be tested
    7773
                ScalarValue::try_new_decimal128(i128::MIN, 10, 5)?,
    7774
                ScalarValue::Decimal256(Some(i256::MIN), 20, 5),
    7775
                // interval, check all possible values
    7776
                ScalarValue::IntervalYearMonth(Some(i32::MIN)),
    7777
                ScalarValue::new_interval_dt(i32::MIN, 999),
    7778
                ScalarValue::new_interval_dt(1, i32::MIN),
    7779
                ScalarValue::new_interval_mdn(i32::MIN, 15, 123_456),
    7780
                ScalarValue::new_interval_mdn(12, i32::MIN, 123_456),
    7781
                ScalarValue::new_interval_mdn(12, 15, i64::MIN),
    7782
                // tz doesn't matter when negating
    7783
                ScalarValue::TimestampSecond(Some(i64::MIN), None),
    7784
                ScalarValue::TimestampMillisecond(Some(i64::MIN), None),
    7785
                ScalarValue::TimestampMicrosecond(Some(i64::MIN), None),
    7786
                ScalarValue::TimestampNanosecond(Some(i64::MIN), None),
    7787
            );
    7788
    7789
            let float_cases = [
    7790
                (
    7791
                    ScalarValue::Float16(Some(f16::MIN)),
    7792
                    ScalarValue::Float16(Some(f16::MAX)),
    7793
                ),
    7794
                (
    7795
                    ScalarValue::Float16(Some(f16::MAX)),
    7796
                    ScalarValue::Float16(Some(f16::MIN)),
    7797
                ),
    7798
                (f32::MIN.into(), f32::MAX.into()),
    7799
                (f32::MAX.into(), f32::MIN.into()),
    7800
                (f64::MIN.into(), f64::MAX.into()),
    7801
                (f64::MAX.into(), f64::MIN.into()),
    7802
            ];
    7803
            // skip float 16 because they aren't supported
    7804
            for (test, expected) in float_cases.into_iter().skip(2) {
    7805
                assert_eq!(test.arithmetic_negate()?, expected);
    7806
            }
    7807
            Ok(())
    7808
        }
    7809
    7810
        #[test]
    7811
        fn f16_test_overflow() {
    7812
            // TODO: if negate supports f16, add these cases to `test_scalar_negative_overflows` test case
    7813
            let cases = [
    7814
                (
    7815
                    ScalarValue::Float16(Some(f16::MIN)),
    7816
                    ScalarValue::Float16(Some(f16::MAX)),
    7817
                ),
    7818
                (
    7819
                    ScalarValue::Float16(Some(f16::MAX)),
    7820
                    ScalarValue::Float16(Some(f16::MIN)),
    7821
                ),
    7822
            ];
    7823
    7824
            for (test, expected) in cases {
    7825
                assert_eq!(test.arithmetic_negate().unwrap(), expected);
    7826
            }
    7827
        }
    7828
    7829
        macro_rules! expect_operation_error {
    7830
            ($TEST_NAME:ident, $FUNCTION:ident, $EXPECTED_ERROR:expr) => {
    7831
                #[test]
    7832
                fn $TEST_NAME() {
    7833
                    let lhs = ScalarValue::UInt64(Some(12));
    7834
                    let rhs = ScalarValue::Int32(Some(-3));
    7835
                    match lhs.$FUNCTION(&rhs) {
    7836
                        Ok(_result) => {
    7837
                            panic!(
    7838
                                "Expected binary operation error between lhs: '{:?}', rhs: {:?}",
    7839
                                lhs, rhs
    7840
                            );
    7841
                        }
    7842
                        Err(e) => {
    7843
                            let error_message = e.to_string();
    7844
                            assert!(
    7845
                                error_message.contains($EXPECTED_ERROR),
    7846
                                "Expected error '{}' not found in actual error '{}'",
    7847
                                $EXPECTED_ERROR,
    7848
                                error_message
    7849
                            );
    7850
                        }
    7851
                    }
    7852
                }
    7853
            };
    7854
        }
    7855
    7856
        expect_operation_error!(
    7857
            expect_add_error,
    7858
            add,
    7859
            "Invalid arithmetic operation: UInt64 + Int32"
    7860
        );
    7861
        expect_operation_error!(
    7862
            expect_sub_error,
    7863
            sub,
    7864
            "Invalid arithmetic operation: UInt64 - Int32"
    7865
        );
    7866
    7867
        macro_rules! decimal_op_test_cases {
    7868
        ($OPERATION:ident, [$([$L_VALUE:expr, $L_PRECISION:expr, $L_SCALE:expr, $R_VALUE:expr, $R_PRECISION:expr, $R_SCALE:expr, $O_VALUE:expr, $O_PRECISION:expr, $O_SCALE:expr]),+]) => {
    7869
                $(
    7870
    7871
                    let left = ScalarValue::Decimal128($L_VALUE, $L_PRECISION, $L_SCALE);
    7872
                    let right = ScalarValue::Decimal128($R_VALUE, $R_PRECISION, $R_SCALE);
    7873
                    let result = left.$OPERATION(&right).unwrap();
    7874
                    assert_eq!(ScalarValue::Decimal128($O_VALUE, $O_PRECISION, $O_SCALE), result);
    7875
    7876
                )+
    7877
            };
    7878
        }
    7879
    7880
        #[test]
    7881
        fn decimal_operations() {
    7882
            decimal_op_test_cases!(
    7883
                add,
    7884
                [
    7885
                    [Some(123), 10, 2, Some(124), 10, 2, Some(123 + 124), 11, 2],
    7886
                    // test sum decimal with diff scale
    7887
                    [
    7888
                        Some(123),
    7889
                        10,
    7890
                        3,
    7891
                        Some(124),
    7892
                        10,
    7893
                        2,
    7894
                        Some(123 + 124 * 10_i128.pow(1)),
    7895
                        12,
    7896
                        3
    7897
                    ],
    7898
                    // diff precision and scale for decimal data type
    7899
                    [
    7900
                        Some(123),
    7901
                        10,
    7902
                        2,
    7903
                        Some(124),
    7904
                        11,
    7905
                        3,
    7906
                        Some(123 * 10_i128.pow(3 - 2) + 124),
    7907
                        12,
    7908
                        3
    7909
                    ]
    7910
                ]
    7911
            );
    7912
        }
    7913
    7914
        #[test]
    7915
        fn decimal_operations_with_nulls() {
    7916
            decimal_op_test_cases!(
    7917
                add,
    7918
                [
    7919
                    // Case: (None, Some, 0)
    7920
                    [None, 10, 2, Some(123), 10, 2, None, 11, 2],
    7921
                    // Case: (Some, None, 0)
    7922
                    [Some(123), 10, 2, None, 10, 2, None, 11, 2],
    7923
                    // Case: (Some, None, _) + Side=False
    7924
                    [Some(123), 8, 2, None, 10, 3, None, 11, 3],
    7925
                    // Case: (None, Some, _) + Side=False
    7926
                    [None, 8, 2, Some(123), 10, 3, None, 11, 3],
    7927
                    // Case: (Some, None, _) + Side=True
    7928
                    [Some(123), 8, 4, None, 10, 3, None, 12, 4],
    7929
                    // Case: (None, Some, _) + Side=True
    7930
                    [None, 10, 3, Some(123), 8, 4, None, 12, 4]
    7931
                ]
    7932
            );
    7933
        }
    7934
    7935
        #[test]
    7936
        fn test_scalar_distance() {
    7937
            let cases = [
    7938
                // scalar (lhs), scalar (rhs), expected distance
    7939
                // ---------------------------------------------
    7940
                (ScalarValue::Int8(Some(1)), ScalarValue::Int8(Some(2)), 1),
    7941
                (ScalarValue::Int8(Some(2)), ScalarValue::Int8(Some(1)), 1),
    7942
                (
    7943
                    ScalarValue::Int16(Some(-5)),
    7944
                    ScalarValue::Int16(Some(5)),
    7945
                    10,
    7946
                ),
    7947
                (
    7948
                    ScalarValue::Int16(Some(5)),
    7949
                    ScalarValue::Int16(Some(-5)),
    7950
                    10,
    7951
                ),
    7952
                (ScalarValue::Int32(Some(0)), ScalarValue::Int32(Some(0)), 0),
    7953
                (
    7954
                    ScalarValue::Int32(Some(-5)),
    7955
                    ScalarValue::Int32(Some(-10)),
    7956
                    5,
    7957
                ),
    7958
                (
    7959
                    ScalarValue::Int64(Some(-10)),
    7960
                    ScalarValue::Int64(Some(-5)),
    7961
                    5,
    7962
                ),
    7963
                (ScalarValue::UInt8(Some(1)), ScalarValue::UInt8(Some(2)), 1),
    7964
                (ScalarValue::UInt8(Some(0)), ScalarValue::UInt8(Some(0)), 0),
    7965
                (
    7966
                    ScalarValue::UInt16(Some(5)),
    7967
                    ScalarValue::UInt16(Some(10)),
    7968
                    5,
    7969
                ),
    7970
                (
    7971
                    ScalarValue::UInt32(Some(10)),
    7972
                    ScalarValue::UInt32(Some(5)),
    7973
                    5,
    7974
                ),
    7975
                (
    7976
                    ScalarValue::UInt64(Some(5)),
    7977
                    ScalarValue::UInt64(Some(10)),
    7978
                    5,
    7979
                ),
    7980
                (
    7981
                    ScalarValue::Float16(Some(f16::from_f32(1.1))),
    7982
                    ScalarValue::Float16(Some(f16::from_f32(1.9))),
    7983
                    1,
    7984
                ),
    7985
                (
    7986
                    ScalarValue::Float16(Some(f16::from_f32(-5.3))),
    7987
                    ScalarValue::Float16(Some(f16::from_f32(-9.2))),
    7988
                    4,
    7989
                ),
    7990
                (
    7991
                    ScalarValue::Float16(Some(f16::from_f32(-5.3))),
    7992
                    ScalarValue::Float16(Some(f16::from_f32(-9.7))),
    7993
                    4,
    7994
                ),
    7995
                (
    7996
                    ScalarValue::Float32(Some(1.0)),
    7997
                    ScalarValue::Float32(Some(2.0)),
    7998
                    1,
    7999
                ),
    8000
                (
    8001
                    ScalarValue::Float32(Some(2.0)),
    8002
                    ScalarValue::Float32(Some(1.0)),
    8003
                    1,
    8004
                ),
    8005
                (
    8006
                    ScalarValue::Float64(Some(0.0)),
    8007
                    ScalarValue::Float64(Some(0.0)),
    8008
                    0,
    8009
                ),
    8010
                (
    8011
                    ScalarValue::Float64(Some(-5.0)),
    8012
                    ScalarValue::Float64(Some(-10.0)),
    8013
                    5,
    8014
                ),
    8015
                (
    8016
                    ScalarValue::Float64(Some(-10.0)),
    8017
                    ScalarValue::Float64(Some(-5.0)),
    8018
                    5,
    8019
                ),
    8020
                // Floats are currently special cased to f64/f32 and the result is rounded
    8021
                // rather than ceiled/floored. In the future we might want to take a mode
    8022
                // which specified the rounding behavior.
    8023
                (
    8024
                    ScalarValue::Float32(Some(1.2)),
    8025
                    ScalarValue::Float32(Some(1.3)),
    8026
                    0,
    8027
                ),
    8028
                (
    8029
                    ScalarValue::Float32(Some(1.1)),
    8030
                    ScalarValue::Float32(Some(1.9)),
    8031
                    1,
    8032
                ),
    8033
                (
    8034
                    ScalarValue::Float64(Some(-5.3)),
    8035
                    ScalarValue::Float64(Some(-9.2)),
    8036
                    4,
    8037
                ),
    8038
                (
    8039
                    ScalarValue::Float64(Some(-5.3)),
    8040
                    ScalarValue::Float64(Some(-9.7)),
    8041
                    4,
    8042
                ),
    8043
                (
    8044
                    ScalarValue::Float64(Some(-5.3)),
    8045
                    ScalarValue::Float64(Some(-9.9)),
    8046
                    5,
    8047
                ),
    8048
                (
    8049
                    ScalarValue::Decimal128(Some(10), 1, 0),
    8050
                    ScalarValue::Decimal128(Some(5), 1, 0),
    8051
                    5,
    8052
                ),
    8053
                (
    8054
                    ScalarValue::Decimal128(Some(5), 1, 0),
    8055
                    ScalarValue::Decimal128(Some(10), 1, 0),
    8056
                    5,
    8057
                ),
    8058
                (
    8059
                    ScalarValue::Decimal256(Some(10.into()), 1, 0),
    8060
                    ScalarValue::Decimal256(Some(5.into()), 1, 0),
    8061
                    5,
    8062
                ),
    8063
                (
    8064
                    ScalarValue::Decimal256(Some(5.into()), 1, 0),
    8065
                    ScalarValue::Decimal256(Some(10.into()), 1, 0),
    8066
                    5,
    8067
                ),
    8068
            ];
    8069
            for (lhs, rhs, expected) in cases.iter() {
    8070
                let distance = lhs.distance(rhs).unwrap();
    8071
                assert_eq!(distance, *expected);
    8072
            }
    8073
        }
    8074
    8075
        #[test]
    8076
        fn test_distance_none() {
    8077
            let cases = [
    8078
                (
    8079
                    ScalarValue::Decimal128(Some(i128::MAX), DECIMAL128_MAX_PRECISION, 0),
    8080
                    ScalarValue::Decimal128(Some(-i128::MAX), DECIMAL128_MAX_PRECISION, 0),
    8081
                ),
    8082
                (
    8083
                    ScalarValue::Decimal256(Some(i256::MAX), DECIMAL256_MAX_PRECISION, 0),
    8084
                    ScalarValue::Decimal256(Some(-i256::MAX), DECIMAL256_MAX_PRECISION, 0),
    8085
                ),
    8086
            ];
    8087
            for (lhs, rhs) in cases.iter() {
    8088
                let distance = lhs.distance(rhs);
    8089
                assert!(distance.is_none(), "{lhs} vs {rhs}");
    8090
            }
    8091
        }
    8092
    8093
        #[test]
    8094
        fn test_scalar_distance_invalid() {
    8095
            let cases = [
    8096
                // scalar (lhs), scalar (rhs)
    8097
                // --------------------------
    8098
                // Same type but with nulls
    8099
                (ScalarValue::Int8(None), ScalarValue::Int8(None)),
    8100
                (ScalarValue::Int8(None), ScalarValue::Int8(Some(1))),
    8101
                (ScalarValue::Int8(Some(1)), ScalarValue::Int8(None)),
    8102
                // Different type
    8103
                (ScalarValue::Int8(Some(1)), ScalarValue::Int16(Some(1))),
    8104
                (ScalarValue::Int8(Some(1)), ScalarValue::Float32(Some(1.0))),
    8105
                (
    8106
                    ScalarValue::Float16(Some(f16::from_f32(1.0))),
    8107
                    ScalarValue::Float32(Some(1.0)),
    8108
                ),
    8109
                (
    8110
                    ScalarValue::Float16(Some(f16::from_f32(1.0))),
    8111
                    ScalarValue::Int32(Some(1)),
    8112
                ),
    8113
                (
    8114
                    ScalarValue::Float64(Some(1.1)),
    8115
                    ScalarValue::Float32(Some(2.2)),
    8116
                ),
    8117
                (
    8118
                    ScalarValue::UInt64(Some(777)),
    8119
                    ScalarValue::Int32(Some(111)),
    8120
                ),
    8121
                // Different types with nulls
    8122
                (ScalarValue::Int8(None), ScalarValue::Int16(Some(1))),
    8123
                (ScalarValue::Int8(Some(1)), ScalarValue::Int16(None)),
    8124
                // Unsupported types
    8125
                (ScalarValue::from("foo"), ScalarValue::from("bar")),
    8126
                (
    8127
                    ScalarValue::Boolean(Some(true)),
    8128
                    ScalarValue::Boolean(Some(false)),
    8129
                ),
    8130
                (ScalarValue::Date32(Some(0)), ScalarValue::Date32(Some(1))),
    8131
                (ScalarValue::Date64(Some(0)), ScalarValue::Date64(Some(1))),
    8132
                (
    8133
                    ScalarValue::Decimal128(Some(123), 5, 5),
    8134
                    ScalarValue::Decimal128(Some(120), 5, 3),
    8135
                ),
    8136
                (
    8137
                    ScalarValue::Decimal128(Some(123), 5, 5),
    8138
                    ScalarValue::Decimal128(Some(120), 3, 5),
    8139
                ),
    8140
                (
    8141
                    ScalarValue::Decimal256(Some(123.into()), 5, 5),
    8142
                    ScalarValue::Decimal256(Some(120.into()), 3, 5),
    8143
                ),
    8144
                // Distance 2 * 2^50 is larger than usize
    8145
                (
    8146
                    ScalarValue::Decimal256(
    8147
                        Some(i256::from_parts(0, 2_i64.pow(50).into())),
    8148
                        1,
    8149
                        0,
    8150
                    ),
    8151
                    ScalarValue::Decimal256(
    8152
                        Some(i256::from_parts(0, (-(2_i64).pow(50)).into())),
    8153
                        1,
    8154
                        0,
    8155
                    ),
    8156
                ),
    8157
                // Distance overflow
    8158
                (
    8159
                    ScalarValue::Decimal256(Some(i256::from_parts(0, i128::MAX)), 1, 0),
    8160
                    ScalarValue::Decimal256(Some(i256::from_parts(0, -i128::MAX)), 1, 0),
    8161
                ),
    8162
            ];
    8163
            for (lhs, rhs) in cases {
    8164
                let distance = lhs.distance(&rhs);
    8165
                assert!(distance.is_none());
    8166
            }
    8167
        }
    8168
    8169
        #[test]
    8170
        fn test_scalar_interval_negate() {
    8171
            let cases = [
    8172
                (
    8173
                    ScalarValue::new_interval_ym(1, 12),
    8174
                    ScalarValue::new_interval_ym(-1, -12),
    8175
                ),
    8176
                (
    8177
                    ScalarValue::new_interval_dt(1, 999),
    8178
                    ScalarValue::new_interval_dt(-1, -999),
    8179
                ),
    8180
                (
    8181
                    ScalarValue::new_interval_mdn(12, 15, 123_456),
    8182
                    ScalarValue::new_interval_mdn(-12, -15, -123_456),
    8183
                ),
    8184
            ];
    8185
            for (expr, expected) in cases.iter() {
    8186
                let result = expr.arithmetic_negate().unwrap();
    8187
                assert_eq!(*expected, result, "-expr:{expr:?}");
    8188
            }
    8189
        }
    8190
    8191
        #[test]
    8192
        fn test_scalar_interval_add() {
    8193
            let cases = [
    8194
                (
    8195
                    ScalarValue::new_interval_ym(1, 12),
    8196
                    ScalarValue::new_interval_ym(1, 12),
    8197
                    ScalarValue::new_interval_ym(2, 24),
    8198
                ),
    8199
                (
    8200
                    ScalarValue::new_interval_dt(1, 999),
    8201
                    ScalarValue::new_interval_dt(1, 999),
    8202
                    ScalarValue::new_interval_dt(2, 1998),
    8203
                ),
    8204
                (
    8205
                    ScalarValue::new_interval_mdn(12, 15, 123_456),
    8206
                    ScalarValue::new_interval_mdn(12, 15, 123_456),
    8207
                    ScalarValue::new_interval_mdn(24, 30, 246_912),
    8208
                ),
    8209
            ];
    8210
            for (lhs, rhs, expected) in cases.iter() {
    8211
                let result = lhs.add(rhs).unwrap();
    8212
                let result_commute = rhs.add(lhs).unwrap();
    8213
                assert_eq!(*expected, result, "lhs:{lhs:?} + rhs:{rhs:?}");
    8214
                assert_eq!(*expected, result_commute, "lhs:{rhs:?} + rhs:{lhs:?}");
    8215
            }
    8216
        }
    8217
    8218
        #[test]
    8219
        fn test_scalar_interval_sub() {
    8220
            let cases = [
    8221
                (
    8222
                    ScalarValue::new_interval_ym(1, 12),
    8223
                    ScalarValue::new_interval_ym(1, 12),
    8224
                    ScalarValue::new_interval_ym(0, 0),
    8225
                ),
    8226
                (
    8227
                    ScalarValue::new_interval_dt(1, 999),
    8228
                    ScalarValue::new_interval_dt(1, 999),
    8229
                    ScalarValue::new_interval_dt(0, 0),
    8230
                ),
    8231
                (
    8232
                    ScalarValue::new_interval_mdn(12, 15, 123_456),
    8233
                    ScalarValue::new_interval_mdn(12, 15, 123_456),
    8234
                    ScalarValue::new_interval_mdn(0, 0, 0),
    8235
                ),
    8236
            ];
    8237
            for (lhs, rhs, expected) in cases.iter() {
    8238
                let result = lhs.sub(rhs).unwrap();
    8239
                assert_eq!(*expected, result, "lhs:{lhs:?} - rhs:{rhs:?}");
    8240
            }
    8241
        }
    8242
    8243
        #[test]
    8244
        fn timestamp_op_random_tests() {
    8245
            // timestamp1 + (or -) interval = timestamp2
    8246
            // timestamp2 - timestamp1 (or timestamp1 - timestamp2) = interval ?
    8247
            let sample_size = 1000;
    8248
            let timestamps1 = get_random_timestamps(sample_size);
    8249
            let intervals = get_random_intervals(sample_size);
    8250
            // ts(sec) + interval(ns) = ts(sec); however,
    8251
            // ts(sec) - ts(sec) cannot be = interval(ns). Therefore,
    8252
            // timestamps are more precise than intervals in tests.
    8253
            for (idx, ts1) in timestamps1.iter().enumerate() {
    8254
                if idx % 2 == 0 {
    8255
                    let timestamp2 = ts1.add(intervals[idx].clone()).unwrap();
    8256
                    let back = timestamp2.sub(intervals[idx].clone()).unwrap();
    8257
                    assert_eq!(ts1, &back);
    8258
                } else {
    8259
                    let timestamp2 = ts1.sub(intervals[idx].clone()).unwrap();
    8260
                    let back = timestamp2.add(intervals[idx].clone()).unwrap();
    8261
                    assert_eq!(ts1, &back);
    8262
                };
    8263
            }
    8264
        }
    8265
    8266
        #[test]
    8267
        fn test_struct_nulls() {
    8268
            let fields_b = Fields::from(vec![
    8269
                Field::new("ba", DataType::UInt64, true),
    8270
                Field::new("bb", DataType::UInt64, true),
    8271
            ]);
    8272
            let fields = Fields::from(vec![
    8273
                Field::new("a", DataType::UInt64, true),
    8274
                Field::new("b", DataType::Struct(fields_b.clone()), true),
    8275
            ]);
    8276
    8277
            let struct_value = vec![
    8278
                (
    8279
                    Arc::clone(&fields[0]),
    8280
                    Arc::new(UInt64Array::from(vec![Some(1)])) as ArrayRef,
    8281
                ),
    8282
                (
    8283
                    Arc::clone(&fields[1]),
    8284
                    Arc::new(StructArray::from(vec![
    8285
                        (
    8286
                            Arc::clone(&fields_b[0]),
    8287
                            Arc::new(UInt64Array::from(vec![Some(2)])) as ArrayRef,
    8288
                        ),
    8289
                        (
    8290
                            Arc::clone(&fields_b[1]),
    8291
                            Arc::new(UInt64Array::from(vec![Some(3)])) as ArrayRef,
    8292
                        ),
    8293
                    ])) as ArrayRef,
    8294
                ),
    8295
            ];
    8296
    8297
            let struct_value_with_nulls = vec![
    8298
                (
    8299
                    Arc::clone(&fields[0]),
    8300
                    Arc::new(UInt64Array::from(vec![Some(1)])) as ArrayRef,
    8301
                ),
    8302
                (
    8303
                    Arc::clone(&fields[1]),
    8304
                    Arc::new(StructArray::from((
    8305
                        vec![
    8306
                            (
    8307
                                Arc::clone(&fields_b[0]),
    8308
                                Arc::new(UInt64Array::from(vec![Some(2)])) as ArrayRef,
    8309
                            ),
    8310
                            (
    8311
                                Arc::clone(&fields_b[1]),
    8312
                                Arc::new(UInt64Array::from(vec![Some(3)])) as ArrayRef,
    8313
                            ),
    8314
                        ],
    8315
                        Buffer::from(&[0]),
    8316
                    ))) as ArrayRef,
    8317
                ),
    8318
            ];
    8319
    8320
            let scalars = vec![
    8321
                // all null
    8322
                ScalarValue::Struct(Arc::new(StructArray::from((
    8323
                    struct_value.clone(),
    8324
                    Buffer::from(&[0]),
    8325
                )))),
    8326
                // field 1 valid, field 2 null
    8327
                ScalarValue::Struct(Arc::new(StructArray::from((
    8328
                    struct_value_with_nulls.clone(),
    8329
                    Buffer::from(&[1]),
    8330
                )))),
    8331
                // all valid
    8332
                ScalarValue::Struct(Arc::new(StructArray::from((
    8333
                    struct_value.clone(),
    8334
                    Buffer::from(&[1]),
    8335
                )))),
    8336
            ];
    8337
    8338
            let check_array = |array| {
    8339
                let is_null = is_null(&array).unwrap();
    8340
                assert_eq!(is_null, BooleanArray::from(vec![true, false, false]));
    8341
    8342
                let formatted = pretty_format_columns("col", &[array]).unwrap().to_string();
    8343
                let formatted = formatted.split('\n').collect::<Vec<_>>();
    8344
                let expected = vec![
    8345
                    "+---------------------------+",
    8346
                    "| col                       |",
    8347
                    "+---------------------------+",
    8348
                    "|                           |",
    8349
                    "| {a: 1, b: }               |",
    8350
                    "| {a: 1, b: {ba: 2, bb: 3}} |",
    8351
                    "+---------------------------+",
    8352
                ];
    8353
                assert_eq!(
    8354
                    formatted, expected,
    8355
                    "Actual:\n{formatted:#?}\n\nExpected:\n{expected:#?}"
    8356
                );
    8357
            };
    8358
    8359
            // test `ScalarValue::iter_to_array`
    8360
            let array = ScalarValue::iter_to_array(scalars.clone()).unwrap();
    8361
            check_array(array);
    8362
    8363
            // test `ScalarValue::to_array` / `ScalarValue::to_array_of_size`
    8364
            let arrays = scalars
    8365
                .iter()
    8366
                .map(ScalarValue::to_array)
    8367
                .collect::<Result<Vec<_>>>()
    8368
                .expect("Failed to convert to array");
    8369
            let arrays = arrays.iter().map(|a| a.as_ref()).collect::<Vec<_>>();
    8370
            let array = arrow::compute::concat(&arrays).unwrap();
    8371
            check_array(array);
    8372
        }
    8373
    8374
        #[test]
    8375
        fn test_struct_display() {
    8376
            let field_a = Field::new("a", DataType::Int32, true);
    8377
            let field_b = Field::new("b", DataType::Utf8, true);
    8378
    8379
            let s = ScalarStructBuilder::new()
    8380
                .with_scalar(field_a, ScalarValue::from(1i32))
    8381
                .with_scalar(field_b, ScalarValue::Utf8(None))
    8382
                .build()
    8383
                .unwrap();
    8384
    8385
            assert_eq!(s.to_string(), "{a:1,b:}");
    8386
            assert_eq!(format!("{s:?}"), r#"Struct({a:1,b:})"#);
    8387
    8388
            let ScalarValue::Struct(arr) = s else {
    8389
                panic!("Expected struct");
    8390
            };
    8391
    8392
            //verify compared to arrow display
    8393
            let batch = RecordBatch::try_from_iter(vec![("s", arr as _)]).unwrap();
    8394
            assert_snapshot!(batches_to_string(&[batch]), @r"
    8395
            +-------------+
    8396
            | s           |
    8397
            +-------------+
    8398
            | {a: 1, b: } |
    8399
            +-------------+
    8400
            ");
    8401
        }
    8402
    8403
        #[test]
    8404
        fn test_null_bug() {
    8405
            let field_a = Field::new("a", DataType::Int32, true);
    8406
            let field_b = Field::new("b", DataType::Int32, true);
    8407
            let fields = Fields::from(vec![field_a, field_b]);
    8408
    8409
            let array_a = Arc::new(Int32Array::from_iter_values([1]));
    8410
            let array_b = Arc::new(Int32Array::from_iter_values([2]));
    8411
            let arrays: Vec<ArrayRef> = vec![array_a, array_b];
    8412
    8413
            let mut not_nulls = NullBufferBuilder::new(1);
    8414
    8415
            not_nulls.append_non_null();
    8416
    8417
            let ar = StructArray::new(fields, arrays, not_nulls.finish());
    8418
            let s = ScalarValue::Struct(Arc::new(ar));
    8419
    8420
            assert_eq!(s.to_string(), "{a:1,b:2}");
    8421
            assert_eq!(format!("{s:?}"), r#"Struct({a:1,b:2})"#);
    8422
    8423
            let ScalarValue::Struct(arr) = s else {
    8424
                panic!("Expected struct");
    8425
            };
    8426
    8427
            //verify compared to arrow display
    8428
            let batch = RecordBatch::try_from_iter(vec![("s", arr as _)]).unwrap();
    8429
            assert_snapshot!(batches_to_string(&[batch]), @r"
    8430
            +--------------+
    8431
            | s            |
    8432
            +--------------+
    8433
            | {a: 1, b: 2} |
    8434
            +--------------+
    8435
            ");
    8436
        }
    8437
    8438
        #[test]
    8439
        fn test_display_date64_large_values() {
    8440
            assert_eq!(
    8441
                format!("{}", ScalarValue::Date64(Some(790179464505))),
    8442
                "1995-01-15"
    8443
            );
    8444
            // This used to panic, see https://github.com/apache/arrow-rs/issues/7728
    8445
            assert_eq!(
    8446
                format!("{}", ScalarValue::Date64(Some(-790179464505600000))),
    8447
                ""
    8448
            );
    8449
        }
    8450
    8451
        #[test]
    8452
        fn test_struct_display_null() {
    8453
            let fields = vec![Field::new("a", DataType::Int32, false)];
    8454
            let s = ScalarStructBuilder::new_null(fields);
    8455
            assert_eq!(s.to_string(), "NULL");
    8456
    8457
            let ScalarValue::Struct(arr) = s else {
    8458
                panic!("Expected struct");
    8459
            };
    8460
    8461
            //verify compared to arrow display
    8462
            let batch = RecordBatch::try_from_iter(vec![("s", arr as _)]).unwrap();
    8463
    8464
            assert_snapshot!(batches_to_string(&[batch]), @r"
    8465
            +---+
    8466
            | s |
    8467
            +---+
    8468
            |   |
    8469
            +---+
    8470
            ");
    8471
        }
    8472
    8473
        #[test]
    8474
        fn test_map_display_and_debug() {
    8475
            let string_builder = StringBuilder::new();
    8476
            let int_builder = Int32Builder::with_capacity(4);
    8477
            let mut builder = MapBuilder::new(None, string_builder, int_builder);
    8478
            builder.keys().append_value("joe");
    8479
            builder.values().append_value(1);
    8480
            builder.append(true).unwrap();
    8481
    8482
            builder.keys().append_value("blogs");
    8483
            builder.values().append_value(2);
    8484
            builder.keys().append_value("foo");
    8485
            builder.values().append_value(4);
    8486
            builder.append(true).unwrap();
    8487
            builder.append(true).unwrap();
    8488
            builder.append(false).unwrap();
    8489
    8490
            let map_value = ScalarValue::Map(Arc::new(builder.finish()));
    8491
    8492
            assert_eq!(map_value.to_string(), "[{joe:1},{blogs:2,foo:4},{},NULL]");
    8493
            assert_eq!(
    8494
                format!("{map_value:?}"),
    8495
                r#"Map([{"joe":"1"},{"blogs":"2","foo":"4"},{},NULL])"#
    8496
            );
    8497
    8498
            let ScalarValue::Map(arr) = map_value else {
    8499
                panic!("Expected map");
    8500
            };
    8501
    8502
            //verify compared to arrow display
    8503
            let batch = RecordBatch::try_from_iter(vec![("m", arr as _)]).unwrap();
    8504
            assert_snapshot!(batches_to_string(&[batch]), @r"
    8505
            +--------------------+
    8506
            | m                  |
    8507
            +--------------------+
    8508
            | {joe: 1}           |
    8509
            | {blogs: 2, foo: 4} |
    8510
            | {}                 |
    8511
            |                    |
    8512
            +--------------------+
    8513
            ");
    8514
        }
    8515
    8516
        #[test]
    8517
        fn test_binary_display() {
    8518
            let no_binary_value = ScalarValue::Binary(None);
    8519
            assert_eq!(format!("{no_binary_value}"), "NULL");
    8520
            let single_binary_value = ScalarValue::Binary(Some(vec![42u8]));
    8521
            assert_eq!(format!("{single_binary_value}"), "2A");
    8522
            let small_binary_value = ScalarValue::Binary(Some(vec![1u8, 2, 3]));
    8523
            assert_eq!(format!("{small_binary_value}"), "010203");
    8524
            let large_binary_value =
    8525
                ScalarValue::Binary(Some(vec![1u8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]));
    8526
            assert_eq!(format!("{large_binary_value}"), "0102030405060708090A...");
    8527
    8528
            let no_binary_value = ScalarValue::BinaryView(None);
    8529
            assert_eq!(format!("{no_binary_value}"), "NULL");
    8530
            let small_binary_value = ScalarValue::BinaryView(Some(vec![1u8, 2, 3]));
    8531
            assert_eq!(format!("{small_binary_value}"), "010203");
    8532
            let large_binary_value =
    8533
                ScalarValue::BinaryView(Some(vec![1u8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]));
    8534
            assert_eq!(format!("{large_binary_value}"), "0102030405060708090A...");
    8535
    8536
            let no_binary_value = ScalarValue::LargeBinary(None);
    8537
            assert_eq!(format!("{no_binary_value}"), "NULL");
    8538
            let small_binary_value = ScalarValue::LargeBinary(Some(vec![1u8, 2, 3]));
    8539
            assert_eq!(format!("{small_binary_value}"), "010203");
    8540
            let large_binary_value =
    8541
                ScalarValue::LargeBinary(Some(vec![1u8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]));
    8542
            assert_eq!(format!("{large_binary_value}"), "0102030405060708090A...");
    8543
    8544
            let no_binary_value = ScalarValue::FixedSizeBinary(3, None);
    8545
            assert_eq!(format!("{no_binary_value}"), "NULL");
    8546
            let small_binary_value = ScalarValue::FixedSizeBinary(3, Some(vec![1u8, 2, 3]));
    8547
            assert_eq!(format!("{small_binary_value}"), "010203");
    8548
            let large_binary_value = ScalarValue::FixedSizeBinary(
    8549
                11,
    8550
                Some(vec![1u8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]),
    8551
            );
    8552
            assert_eq!(format!("{large_binary_value}"), "0102030405060708090A...");
    8553
        }
    8554
    8555
        #[test]
    8556
        fn test_binary_debug() {
    8557
            let no_binary_value = ScalarValue::Binary(None);
    8558
            assert_eq!(format!("{no_binary_value:?}"), "Binary(NULL)");
    8559
            let single_binary_value = ScalarValue::Binary(Some(vec![42u8]));
    8560
            assert_eq!(format!("{single_binary_value:?}"), "Binary(\"42\")");
    8561
            let small_binary_value = ScalarValue::Binary(Some(vec![1u8, 2, 3]));
    8562
            assert_eq!(format!("{small_binary_value:?}"), "Binary(\"1,2,3\")");
    8563
            let large_binary_value =
    8564
                ScalarValue::Binary(Some(vec![1u8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]));
    8565
            assert_eq!(
    8566
                format!("{large_binary_value:?}"),
    8567
                "Binary(\"1,2,3,4,5,6,7,8,9,10,11\")"
    8568
            );
    8569
    8570
            let no_binary_value = ScalarValue::BinaryView(None);
    8571
            assert_eq!(format!("{no_binary_value:?}"), "BinaryView(NULL)");
    8572
            let small_binary_value = ScalarValue::BinaryView(Some(vec![1u8, 2, 3]));
    8573
            assert_eq!(format!("{small_binary_value:?}"), "BinaryView(\"1,2,3\")");
    8574
            let large_binary_value =
    8575
                ScalarValue::BinaryView(Some(vec![1u8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]));
    8576
            assert_eq!(
    8577
                format!("{large_binary_value:?}"),
    8578
                "BinaryView(\"1,2,3,4,5,6,7,8,9,10,11\")"
    8579
            );
    8580
    8581
            let no_binary_value = ScalarValue::LargeBinary(None);
    8582
            assert_eq!(format!("{no_binary_value:?}"), "LargeBinary(NULL)");
    8583
            let small_binary_value = ScalarValue::LargeBinary(Some(vec![1u8, 2, 3]));
    8584
            assert_eq!(format!("{small_binary_value:?}"), "LargeBinary(\"1,2,3\")");
    8585
            let large_binary_value =
    8586
                ScalarValue::LargeBinary(Some(vec![1u8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]));
    8587
            assert_eq!(
    8588
                format!("{large_binary_value:?}"),
    8589
                "LargeBinary(\"1,2,3,4,5,6,7,8,9,10,11\")"
    8590
            );
    8591
    8592
            let no_binary_value = ScalarValue::FixedSizeBinary(3, None);
    8593
            assert_eq!(format!("{no_binary_value:?}"), "FixedSizeBinary(3, NULL)");
    8594
            let small_binary_value = ScalarValue::FixedSizeBinary(3, Some(vec![1u8, 2, 3]));
    8595
            assert_eq!(
    8596
                format!("{small_binary_value:?}"),
    8597
                "FixedSizeBinary(3, \"1,2,3\")"
    8598
            );
    8599
            let large_binary_value = ScalarValue::FixedSizeBinary(
    8600
                11,
    8601
                Some(vec![1u8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]),
    8602
            );
    8603
            assert_eq!(
    8604
                format!("{large_binary_value:?}"),
    8605
                "FixedSizeBinary(11, \"1,2,3,4,5,6,7,8,9,10,11\")"
    8606
            );
    8607
        }
    8608
    8609
        #[test]
    8610
        fn test_build_timestamp_millisecond_list() {
    8611
            let values = vec![ScalarValue::TimestampMillisecond(Some(1), None)];
    8612
            let arr = ScalarValue::new_list_nullable(
    8613
                &values,
    8614
                &DataType::Timestamp(TimeUnit::Millisecond, None),
    8615
            );
    8616
            assert_eq!(1, arr.len());
    8617
        }
    8618
    8619
        #[test]
    8620
        fn test_newlist_timestamp_zone() {
    8621
            let s: &'static str = "UTC";
    8622
            let values = vec![ScalarValue::TimestampMillisecond(Some(1), Some(s.into()))];
    8623
            let arr = ScalarValue::new_list_nullable(
    8624
                &values,
    8625
                &DataType::Timestamp(TimeUnit::Millisecond, Some(s.into())),
    8626
            );
    8627
            assert_eq!(1, arr.len());
    8628
            assert_eq!(
    8629
                arr.data_type(),
    8630
                &DataType::List(Arc::new(Field::new_list_field(
    8631
                    DataType::Timestamp(TimeUnit::Millisecond, Some(s.into())),
    8632
                    true,
    8633
                )))
    8634
            );
    8635
        }
    8636
    8637
        fn get_random_timestamps(sample_size: u64) -> Vec<ScalarValue> {
    8638
            let vector_size = sample_size;
    8639
            let mut timestamp = vec![];
    8640
            let mut rng = rand::rng();
    8641
            for i in 0..vector_size {
    8642
                let year = rng.random_range(1995..=2050);
    8643
                let month = rng.random_range(1..=12);
    8644
                let day = rng.random_range(1..=28); // to exclude invalid dates
    8645
                let hour = rng.random_range(0..=23);
    8646
                let minute = rng.random_range(0..=59);
    8647
                let second = rng.random_range(0..=59);
    8648
                if i % 4 == 0 {
    8649
                    timestamp.push(ScalarValue::TimestampSecond(
    8650
                        Some(
    8651
                            NaiveDate::from_ymd_opt(year, month, day)
    8652
                                .unwrap()
    8653
                                .and_hms_opt(hour, minute, second)
    8654
                                .unwrap()
    8655
                                .and_utc()
    8656
                                .timestamp(),
    8657
                        ),
    8658
                        None,
    8659
                    ))
    8660
                } else if i % 4 == 1 {
    8661
                    let millisec = rng.random_range(0..=999);
    8662
                    timestamp.push(ScalarValue::TimestampMillisecond(
    8663
                        Some(
    8664
                            NaiveDate::from_ymd_opt(year, month, day)
    8665
                                .unwrap()
    8666
                                .and_hms_milli_opt(hour, minute, second, millisec)
    8667
                                .unwrap()
    8668
                                .and_utc()
    8669
                                .timestamp_millis(),
    8670
                        ),
    8671
                        None,
    8672
                    ))
    8673
                } else if i % 4 == 2 {
    8674
                    let microsec = rng.random_range(0..=999_999);
    8675
                    timestamp.push(ScalarValue::TimestampMicrosecond(
    8676
                        Some(
    8677
                            NaiveDate::from_ymd_opt(year, month, day)
    8678
                                .unwrap()
    8679
                                .and_hms_micro_opt(hour, minute, second, microsec)
    8680
                                .unwrap()
    8681
                                .and_utc()
    8682
                                .timestamp_micros(),
    8683
                        ),
    8684
                        None,
    8685
                    ))
    8686
                } else if i % 4 == 3 {
    8687
                    let nanosec = rng.random_range(0..=999_999_999);
    8688
                    timestamp.push(ScalarValue::TimestampNanosecond(
    8689
                        Some(
    8690
                            NaiveDate::from_ymd_opt(year, month, day)
    8691
                                .unwrap()
    8692
                                .and_hms_nano_opt(hour, minute, second, nanosec)
    8693
                                .unwrap()
    8694
                                .and_utc()
    8695
                                .timestamp_nanos_opt()
    8696
                                .unwrap(),
    8697
                        ),
    8698
                        None,
    8699
                    ))
    8700
                }
    8701
            }
    8702
            timestamp
    8703
        }
    8704
    8705
        fn get_random_intervals(sample_size: u64) -> Vec<ScalarValue> {
    8706
            const MILLISECS_IN_ONE_DAY: i64 = 86_400_000;
    8707
            const NANOSECS_IN_ONE_DAY: i64 = 86_400_000_000_000;
    8708
    8709
            let vector_size = sample_size;
    8710
            let mut intervals = vec![];
    8711
            let mut rng = rand::rng();
    8712
            const SECS_IN_ONE_DAY: i32 = 86_400;
    8713
            const MICROSECS_IN_ONE_DAY: i64 = 86_400_000_000;
    8714
            for i in 0..vector_size {
    8715
                if i % 4 == 0 {
    8716
                    let days = rng.random_range(0..5000);
    8717
                    // to not break second precision
    8718
                    let millis = rng.random_range(0..SECS_IN_ONE_DAY) * 1000;
    8719
                    intervals.push(ScalarValue::new_interval_dt(days, millis));
    8720
                } else if i % 4 == 1 {
    8721
                    let days = rng.random_range(0..5000);
    8722
                    let millisec = rng.random_range(0..(MILLISECS_IN_ONE_DAY as i32));
    8723
                    intervals.push(ScalarValue::new_interval_dt(days, millisec));
    8724
                } else if i % 4 == 2 {
    8725
                    let days = rng.random_range(0..5000);
    8726
                    // to not break microsec precision
    8727
                    let nanosec = rng.random_range(0..MICROSECS_IN_ONE_DAY) * 1000;
    8728
                    intervals.push(ScalarValue::new_interval_mdn(0, days, nanosec));
    8729
                } else {
    8730
                    let days = rng.random_range(0..5000);
    8731
                    let nanosec = rng.random_range(0..NANOSECS_IN_ONE_DAY);
    8732
                    intervals.push(ScalarValue::new_interval_mdn(0, days, nanosec));
    8733
                }
    8734
            }
    8735
            intervals
    8736
        }
    8737
    8738
        fn union_fields() -> UnionFields {
    8739
            [
    8740
                (0, Arc::new(Field::new("A", DataType::Int32, true))),
    8741
                (1, Arc::new(Field::new("B", DataType::Float64, true))),
    8742
            ]
    8743
            .into_iter()
    8744
            .collect()
    8745
        }
    8746
    8747
        #[test]
    8748
        fn sparse_scalar_union_is_null() {
    8749
            let sparse_scalar = ScalarValue::Union(
    8750
                Some((0_i8, Box::new(ScalarValue::Int32(None)))),
    8751
                union_fields(),
    8752
                UnionMode::Sparse,
    8753
            );
    8754
            assert!(sparse_scalar.is_null());
    8755
        }
    8756
    8757
        #[test]
    8758
        fn dense_scalar_union_is_null() {
    8759
            let dense_scalar = ScalarValue::Union(
    8760
                Some((0_i8, Box::new(ScalarValue::Int32(None)))),
    8761
                union_fields(),
    8762
                UnionMode::Dense,
    8763
            );
    8764
            assert!(dense_scalar.is_null());
    8765
        }
    8766
    8767
        #[test]
    8768
        fn cast_date_to_timestamp_overflow_returns_error() {
    8769
            let scalar = ScalarValue::Date32(Some(i32::MAX));
    8770
            let err = scalar
    8771
                .cast_to(&DataType::Timestamp(TimeUnit::Nanosecond, None))
    8772
                .expect_err("expected cast to fail");
    8773
            assert!(
    8774
                err.to_string()
    8775
                    .contains("converted value exceeds the representable i64 range"),
    8776
                "unexpected error: {err}"
    8777
            );
    8778
        }
    8779
    8780
        #[test]
    8781
        fn null_dictionary_scalar_produces_null_dictionary_array() {
    8782
            let dictionary_scalar = ScalarValue::Dictionary(
    8783
                Box::new(DataType::Int32),
    8784
                Box::new(ScalarValue::Null),
    8785
            );
    8786
            assert!(dictionary_scalar.is_null());
    8787
            let dictionary_array = dictionary_scalar.to_array().unwrap();
    8788
            assert!(dictionary_array.is_null(0));
    8789
        }
    8790
    8791
        #[test]
    8792
        fn test_scalar_value_try_new_null() {
    8793
            let scalars = vec![
    8794
                ScalarValue::try_new_null(&DataType::Boolean).unwrap(),
    8795
                ScalarValue::try_new_null(&DataType::Int8).unwrap(),
    8796
                ScalarValue::try_new_null(&DataType::Int16).unwrap(),
    8797
                ScalarValue::try_new_null(&DataType::Int32).unwrap(),
    8798
                ScalarValue::try_new_null(&DataType::Int64).unwrap(),
    8799
                ScalarValue::try_new_null(&DataType::UInt8).unwrap(),
    8800
                ScalarValue::try_new_null(&DataType::UInt16).unwrap(),
    8801
                ScalarValue::try_new_null(&DataType::UInt32).unwrap(),
    8802
                ScalarValue::try_new_null(&DataType::UInt64).unwrap(),
    8803
                ScalarValue::try_new_null(&DataType::Float16).unwrap(),
    8804
                ScalarValue::try_new_null(&DataType::Float32).unwrap(),
    8805
                ScalarValue::try_new_null(&DataType::Float64).unwrap(),
    8806
                ScalarValue::try_new_null(&DataType::Decimal128(42, 42)).unwrap(),
    8807
                ScalarValue::try_new_null(&DataType::Decimal256(42, 42)).unwrap(),
    8808
                ScalarValue::try_new_null(&DataType::Utf8).unwrap(),
    8809
                ScalarValue::try_new_null(&DataType::LargeUtf8).unwrap(),
    8810
                ScalarValue::try_new_null(&DataType::Utf8View).unwrap(),
    8811
                ScalarValue::try_new_null(&DataType::Binary).unwrap(),
    8812
                ScalarValue::try_new_null(&DataType::BinaryView).unwrap(),
    8813
                ScalarValue::try_new_null(&DataType::FixedSizeBinary(42)).unwrap(),
    8814
                ScalarValue::try_new_null(&DataType::LargeBinary).unwrap(),
    8815
                ScalarValue::try_new_null(&DataType::Date32).unwrap(),
    8816
                ScalarValue::try_new_null(&DataType::Date64).unwrap(),
    8817
                ScalarValue::try_new_null(&DataType::Time32(TimeUnit::Second)).unwrap(),
    8818
                ScalarValue::try_new_null(&DataType::Time32(TimeUnit::Millisecond)).unwrap(),
    8819
                ScalarValue::try_new_null(&DataType::Time64(TimeUnit::Microsecond)).unwrap(),
    8820
                ScalarValue::try_new_null(&DataType::Time64(TimeUnit::Nanosecond)).unwrap(),
    8821
                ScalarValue::try_new_null(&DataType::Timestamp(TimeUnit::Second, None))
    8822
                    .unwrap(),
    8823
                ScalarValue::try_new_null(&DataType::Timestamp(TimeUnit::Millisecond, None))
    8824
                    .unwrap(),
    8825
                ScalarValue::try_new_null(&DataType::Timestamp(TimeUnit::Microsecond, None))
    8826
                    .unwrap(),
    8827
                ScalarValue::try_new_null(&DataType::Timestamp(TimeUnit::Nanosecond, None))
    8828
                    .unwrap(),
    8829
                ScalarValue::try_new_null(&DataType::Interval(IntervalUnit::YearMonth))
    8830
                    .unwrap(),
    8831
                ScalarValue::try_new_null(&DataType::Interval(IntervalUnit::DayTime))
    8832
                    .unwrap(),
    8833
                ScalarValue::try_new_null(&DataType::Interval(IntervalUnit::MonthDayNano))
    8834
                    .unwrap(),
    8835
                ScalarValue::try_new_null(&DataType::Duration(TimeUnit::Second)).unwrap(),
    8836
                ScalarValue::try_new_null(&DataType::Duration(TimeUnit::Microsecond))
    8837
                    .unwrap(),
    8838
                ScalarValue::try_new_null(&DataType::Duration(TimeUnit::Nanosecond)).unwrap(),
    8839
                ScalarValue::try_new_null(&DataType::Null).unwrap(),
    8840
            ];
    8841
            assert!(scalars.iter().all(|s| s.is_null()));
    8842
    8843
            let field_ref = Arc::new(Field::new("foo", DataType::Int32, true));
    8844
            let map_field_ref = Arc::new(Field::new(
    8845
                "foo",
    8846
                DataType::Struct(Fields::from(vec![
    8847
                    Field::new("bar", DataType::Utf8, true),
    8848
                    Field::new("baz", DataType::Int32, true),
    8849
                ])),
    8850
                true,
    8851
            ));
    8852
            let scalars = [
    8853
                ScalarValue::try_new_null(&DataType::List(Arc::clone(&field_ref))).unwrap(),
    8854
                ScalarValue::try_new_null(&DataType::LargeList(Arc::clone(&field_ref)))
    8855
                    .unwrap(),
    8856
                ScalarValue::try_new_null(&DataType::FixedSizeList(
    8857
                    Arc::clone(&field_ref),
    8858
                    42,
    8859
                ))
    8860
                .unwrap(),
    8861
                ScalarValue::try_new_null(&DataType::Struct(
    8862
                    vec![Arc::clone(&field_ref)].into(),
    8863
                ))
    8864
                .unwrap(),
    8865
                ScalarValue::try_new_null(&DataType::Map(map_field_ref, false)).unwrap(),
    8866
                ScalarValue::try_new_null(&DataType::Union(
    8867
                    UnionFields::new(vec![42], vec![field_ref]),
    8868
                    UnionMode::Dense,
    8869
                ))
    8870
                .unwrap(),
    8871
            ];
    8872
            assert!(scalars.iter().all(|s| s.is_null()));
    8873
        }
    8874
    8875
        // `err.to_string()` depends on backtrace being present (may have backtrace appended)
    8876
        // `err.strip_backtrace()` also depends on backtrace being present (may have "This was likely caused by ..." stripped)
    8877
        fn assert_starts_with(actual: impl AsRef<str>, expected_prefix: impl AsRef<str>) {
    8878
            let actual = actual.as_ref();
    8879
            let expected_prefix = expected_prefix.as_ref();
    8880
            assert!(
    8881
                actual.starts_with(expected_prefix),
    8882
                "Expected '{actual}' to start with '{expected_prefix}'"
    8883
            );
    8884
        }
    8885
    8886
        #[test]
    8887
        fn test_new_default() {
    8888
            // Test numeric types
    8889
            assert_eq!(
    8890
                ScalarValue::new_default(&DataType::Int32).unwrap(),
    8891
                ScalarValue::Int32(Some(0))
    8892
            );
    8893
            assert_eq!(
    8894
                ScalarValue::new_default(&DataType::Float64).unwrap(),
    8895
                ScalarValue::Float64(Some(0.0))
    8896
            );
    8897
            assert_eq!(
    8898
                ScalarValue::new_default(&DataType::Boolean).unwrap(),
    8899
                ScalarValue::Boolean(Some(false))
    8900
            );
    8901
    8902
            // Test string types
    8903
            assert_eq!(
    8904
                ScalarValue::new_default(&DataType::Utf8).unwrap(),
    8905
                ScalarValue::Utf8(Some("".to_string()))
    8906
            );
    8907
            assert_eq!(
    8908
                ScalarValue::new_default(&DataType::LargeUtf8).unwrap(),
    8909
                ScalarValue::LargeUtf8(Some("".to_string()))
    8910
            );
    8911
    8912
            // Test binary types
    8913
            assert_eq!(
    8914
                ScalarValue::new_default(&DataType::Binary).unwrap(),
    8915
                ScalarValue::Binary(Some(vec![]))
    8916
            );
    8917
    8918
            // Test fixed size binary
    8919
            assert_eq!(
    8920
                ScalarValue::new_default(&DataType::FixedSizeBinary(5)).unwrap(),
    8921
                ScalarValue::FixedSizeBinary(5, Some(vec![0, 0, 0, 0, 0]))
    8922
            );
    8923
    8924
            // Test temporal types
    8925
            assert_eq!(
    8926
                ScalarValue::new_default(&DataType::Date32).unwrap(),
    8927
                ScalarValue::Date32(Some(0))
    8928
            );
    8929
            assert_eq!(
    8930
                ScalarValue::new_default(&DataType::Time32(TimeUnit::Second)).unwrap(),
    8931
                ScalarValue::Time32Second(Some(0))
    8932
            );
    8933
    8934
            // Test decimal types
    8935
            assert_eq!(
    8936
                ScalarValue::new_default(&DataType::Decimal128(10, 2)).unwrap(),
    8937
                ScalarValue::Decimal128(Some(0), 10, 2)
    8938
            );
    8939
    8940
            // Test list type
    8941
            let list_field = Field::new_list_field(DataType::Int32, true);
    8942
            let list_result =
    8943
                ScalarValue::new_default(&DataType::List(Arc::new(list_field.clone())))
    8944
                    .unwrap();
    8945
            match list_result {
    8946
                ScalarValue::List(arr) => {
    8947
                    assert_eq!(arr.len(), 1);
    8948
                    assert_eq!(arr.value_length(0), 0); // empty list
    8949
                }
    8950
                _ => panic!("Expected List"),
    8951
            }
    8952
    8953
            // Test struct type
    8954
            let struct_fields = Fields::from(vec![
    8955
                Field::new("a", DataType::Int32, false),
    8956
                Field::new("b", DataType::Utf8, false),
    8957
            ]);
    8958
            let struct_result =
    8959
                ScalarValue::new_default(&DataType::Struct(struct_fields.clone())).unwrap();
    8960
            match struct_result {
    8961
                ScalarValue::Struct(arr) => {
    8962
                    assert_eq!(arr.len(), 1);
    8963
                    assert_eq!(arr.column(0).as_primitive::<Int32Type>().value(0), 0);
    8964
                    assert_eq!(arr.column(1).as_string::<i32>().value(0), "");
    8965
                }
    8966
                _ => panic!("Expected Struct"),
    8967
            }
    8968
    8969
            // Test union type
    8970
            let union_fields = UnionFields::new(
    8971
                vec![0, 1],
    8972
                vec![
    8973
                    Field::new("i32", DataType::Int32, false),
    8974
                    Field::new("f64", DataType::Float64, false),
    8975
                ],
    8976
            );
    8977
            let union_result = ScalarValue::new_default(&DataType::Union(
    8978
                union_fields.clone(),
    8979
                UnionMode::Sparse,
    8980
            ))
    8981
            .unwrap();
    8982
            match union_result {
    8983
                ScalarValue::Union(Some((type_id, value)), _, _) => {
    8984
                    assert_eq!(type_id, 0);
    8985
                    assert_eq!(*value, ScalarValue::Int32(Some(0)));
    8986
                }
    8987
                _ => panic!("Expected Union"),
    8988
            }
    8989
        }
    8990
    8991
        #[test]
    8992
        fn test_scalar_min() {
    8993
            // Test integer types
    8994
            assert_eq!(
    8995
                ScalarValue::min(&DataType::Int8),
    8996
                Some(ScalarValue::Int8(Some(i8::MIN)))
    8997
            );
    8998
            assert_eq!(
    8999
                ScalarValue::min(&DataType::Int32),
    9000
                Some(ScalarValue::Int32(Some(i32::MIN)))
    9001
            );
    9002
            assert_eq!(
    9003
                ScalarValue::min(&DataType::UInt8),
    9004
                Some(ScalarValue::UInt8(Some(0)))
    9005
            );
    9006
            assert_eq!(
    9007
                ScalarValue::min(&DataType::UInt64),
    9008
                Some(ScalarValue::UInt64(Some(0)))
    9009
            );
    9010
    9011
            // Test float types
    9012
            assert_eq!(
    9013
                ScalarValue::min(&DataType::Float32),
    9014
                Some(ScalarValue::Float32(Some(f32::NEG_INFINITY)))
    9015
            );
    9016
            assert_eq!(
    9017
                ScalarValue::min(&DataType::Float64),
    9018
                Some(ScalarValue::Float64(Some(f64::NEG_INFINITY)))
    9019
            );
    9020
    9021
            // Test decimal types
    9022
            let decimal_min = ScalarValue::min(&DataType::Decimal128(5, 2)).unwrap();
    9023
            match decimal_min {
    9024
                ScalarValue::Decimal128(Some(val), 5, 2) => {
    9025
                    assert_eq!(val, -99999); // -999.99 with scale 2
    9026
                }
    9027
                _ => panic!("Expected Decimal128"),
    9028
            }
    9029
    9030
            // Test temporal types
    9031
            assert_eq!(
    9032
                ScalarValue::min(&DataType::Date32),
    9033
                Some(ScalarValue::Date32(Some(i32::MIN)))
    9034
            );
    9035
            assert_eq!(
    9036
                ScalarValue::min(&DataType::Time32(TimeUnit::Second)),
    9037
                Some(ScalarValue::Time32Second(Some(0)))
    9038
            );
    9039
            assert_eq!(
    9040
                ScalarValue::min(&DataType::Timestamp(TimeUnit::Nanosecond, None)),
    9041
                Some(ScalarValue::TimestampNanosecond(Some(i64::MIN), None))
    9042
            );
    9043
    9044
            // Test duration types
    9045
            assert_eq!(
    9046
                ScalarValue::min(&DataType::Duration(TimeUnit::Second)),
    9047
                Some(ScalarValue::DurationSecond(Some(i64::MIN)))
    9048
            );
    9049
    9050
            // Test unsupported types
    9051
            assert_eq!(ScalarValue::min(&DataType::Utf8), None);
    9052
            assert_eq!(ScalarValue::min(&DataType::Binary), None);
    9053
            assert_eq!(
    9054
                ScalarValue::min(&DataType::List(Arc::new(Field::new(
    9055
                    "item",
    9056
                    DataType::Int32,
    9057
                    true
    9058
                )))),
    9059
                None
    9060
            );
    9061
        }
    9062
    9063
        #[test]
    9064
        fn test_scalar_max() {
    9065
            // Test integer types
    9066
            assert_eq!(
    9067
                ScalarValue::max(&DataType::Int8),
    9068
                Some(ScalarValue::Int8(Some(i8::MAX)))
    9069
            );
    9070
            assert_eq!(
    9071
                ScalarValue::max(&DataType::Int32),
    9072
                Some(ScalarValue::Int32(Some(i32::MAX)))
    9073
            );
    9074
            assert_eq!(
    9075
                ScalarValue::max(&DataType::UInt8),
    9076
                Some(ScalarValue::UInt8(Some(u8::MAX)))
    9077
            );
    9078
            assert_eq!(
    9079
                ScalarValue::max(&DataType::UInt64),
    9080
                Some(ScalarValue::UInt64(Some(u64::MAX)))
    9081
            );
    9082
    9083
            // Test float types
    9084
            assert_eq!(
    9085
                ScalarValue::max(&DataType::Float32),
    9086
                Some(ScalarValue::Float32(Some(f32::INFINITY)))
    9087
            );
    9088
            assert_eq!(
    9089
                ScalarValue::max(&DataType::Float64),
    9090
                Some(ScalarValue::Float64(Some(f64::INFINITY)))
    9091
            );
    9092
    9093
            // Test decimal types
    9094
            let decimal_max = ScalarValue::max(&DataType::Decimal128(5, 2)).unwrap();
    9095
            match decimal_max {
    9096
                ScalarValue::Decimal128(Some(val), 5, 2) => {
    9097
                    assert_eq!(val, 99999); // 999.99 with scale 2
    9098
                }
    9099
                _ => panic!("Expected Decimal128"),
    9100
            }
    9101
    9102
            // Test temporal types
    9103
            assert_eq!(
    9104
                ScalarValue::max(&DataType::Date32),
    9105
                Some(ScalarValue::Date32(Some(i32::MAX)))
    9106
            );
    9107
            assert_eq!(
    9108
                ScalarValue::max(&DataType::Time32(TimeUnit::Second)),
    9109
                Some(ScalarValue::Time32Second(Some(86_399))) // 23:59:59
    9110
            );
    9111
            assert_eq!(
    9112
                ScalarValue::max(&DataType::Time64(TimeUnit::Microsecond)),
    9113
                Some(ScalarValue::Time64Microsecond(Some(86_399_999_999))) // 23:59:59.999999
    9114
            );
    9115
            assert_eq!(
    9116
                ScalarValue::max(&DataType::Timestamp(TimeUnit::Nanosecond, None)),
    9117
                Some(ScalarValue::TimestampNanosecond(Some(i64::MAX), None))
    9118
            );
    9119
    9120
            // Test duration types
    9121
            assert_eq!(
    9122
                ScalarValue::max(&DataType::Duration(TimeUnit::Millisecond)),
    9123
                Some(ScalarValue::DurationMillisecond(Some(i64::MAX)))
    9124
            );
    9125
    9126
            // Test unsupported types
    9127
            assert_eq!(ScalarValue::max(&DataType::Utf8), None);
    9128
            assert_eq!(ScalarValue::max(&DataType::Binary), None);
    9129
            assert_eq!(
    9130
                ScalarValue::max(&DataType::Struct(Fields::from(vec![Field::new(
    9131
                    "field",
    9132
                    DataType::Int32,
    9133
                    true
    9134
                )]))),
    9135
                None
    9136
            );
    9137
        }
    9138
    9139
        #[test]
    9140
        fn test_min_max_float16() {
    9141
            // Test Float16 min and max
    9142
            let min_f16 = ScalarValue::min(&DataType::Float16).unwrap();
    9143
            match min_f16 {
    9144
                ScalarValue::Float16(Some(val)) => {
    9145
                    assert_eq!(val, f16::NEG_INFINITY);
    9146
                }
    9147
                _ => panic!("Expected Float16"),
    9148
            }
    9149
    9150
            let max_f16 = ScalarValue::max(&DataType::Float16).unwrap();
    9151
            match max_f16 {
    9152
                ScalarValue::Float16(Some(val)) => {
    9153
                    assert_eq!(val, f16::INFINITY);
    9154
                }
    9155
                _ => panic!("Expected Float16"),
    9156
            }
    9157
        }
    9158
    9159
        #[test]
    9160
        fn test_new_default_interval() {
    9161
            // Test all interval types
    9162
            assert_eq!(
    9163
                ScalarValue::new_default(&DataType::Interval(IntervalUnit::YearMonth))
    9164
                    .unwrap(),
    9165
                ScalarValue::IntervalYearMonth(Some(0))
    9166
            );
    9167
            assert_eq!(
    9168
                ScalarValue::new_default(&DataType::Interval(IntervalUnit::DayTime)).unwrap(),
    9169
                ScalarValue::IntervalDayTime(Some(IntervalDayTime::ZERO))
    9170
            );
    9171
            assert_eq!(
    9172
                ScalarValue::new_default(&DataType::Interval(IntervalUnit::MonthDayNano))
    9173
                    .unwrap(),
    9174
                ScalarValue::IntervalMonthDayNano(Some(IntervalMonthDayNano::ZERO))
    9175
            );
    9176
        }
    9177
    9178
        #[test]
    9179
        fn test_min_max_with_timezone() {
    9180
            let tz = Some(Arc::from("UTC"));
    9181
    9182
            // Test timestamp with timezone
    9183
            let min_ts =
    9184
                ScalarValue::min(&DataType::Timestamp(TimeUnit::Second, tz.clone())).unwrap();
    9185
            match min_ts {
    9186
                ScalarValue::TimestampSecond(Some(val), Some(tz_str)) => {
    9187
                    assert_eq!(val, i64::MIN);
    9188
                    assert_eq!(tz_str.as_ref(), "UTC");
    9189
                }
    9190
                _ => panic!("Expected TimestampSecond with timezone"),
    9191
            }
    9192
    9193
            let max_ts =
    9194
                ScalarValue::max(&DataType::Timestamp(TimeUnit::Millisecond, tz.clone()))
    9195
                    .unwrap();
    9196
            match max_ts {
    9197
                ScalarValue::TimestampMillisecond(Some(val), Some(tz_str)) => {
    9198
                    assert_eq!(val, i64::MAX);
    9199
                    assert_eq!(tz_str.as_ref(), "UTC");
    9200
                }
    9201
                _ => panic!("Expected TimestampMillisecond with timezone"),
    9202
            }
    9203
        }
    9204
    9205
        #[test]
    9206
        fn test_convert_array_to_scalar_vec() {
    9207
            // 1: Regular ListArray
    9208
            let list = ListArray::from_iter_primitive::<Int64Type, _, _>(vec![
    9209
                Some(vec![Some(1), Some(2)]),
    9210
                None,
    9211
                Some(vec![Some(3), None, Some(4)]),
    9212
            ]);
    9213
            let converted = ScalarValue::convert_array_to_scalar_vec(&list).unwrap();
    9214
            assert_eq!(
    9215
                converted,
    9216
                vec![
    9217
                    Some(vec![
    9218
                        ScalarValue::Int64(Some(1)),
    9219
                        ScalarValue::Int64(Some(2))
    9220
                    ]),
    9221
                    None,
    9222
                    Some(vec![
    9223
                        ScalarValue::Int64(Some(3)),
    9224
                        ScalarValue::Int64(None),
    9225
                        ScalarValue::Int64(Some(4))
    9226
                    ]),
    9227
                ]
    9228
            );
    9229
    9230
            // 2: Regular LargeListArray
    9231
            let large_list = LargeListArray::from_iter_primitive::<Int64Type, _, _>(vec![
    9232
                Some(vec![Some(1), Some(2)]),
    9233
                None,
    9234
                Some(vec![Some(3), None, Some(4)]),
    9235
            ]);
    9236
            let converted = ScalarValue::convert_array_to_scalar_vec(&large_list).unwrap();
    9237
            assert_eq!(
    9238
                converted,
    9239
                vec![
    9240
                    Some(vec![
    9241
                        ScalarValue::Int64(Some(1)),
    9242
                        ScalarValue::Int64(Some(2))
    9243
                    ]),
    9244
                    None,
    9245
                    Some(vec![
    9246
                        ScalarValue::Int64(Some(3)),
    9247
                        ScalarValue::Int64(None),
    9248
                        ScalarValue::Int64(Some(4))
    9249
                    ]),
    9250
                ]
    9251
            );
    9252
    9253
            // 3: Funky (null slot has non-zero list offsets)
    9254
            // Offsets + Values looks like this: [[1, 2], [3, 4], [5]]
    9255
            // But with NullBuffer it's like this: [[1, 2], NULL, [5]]
    9256
            let funky = ListArray::new(
    9257
                Field::new_list_field(DataType::Int64, true).into(),
    9258
                OffsetBuffer::new(vec![0, 2, 4, 5].into()),
    9259
                Arc::new(Int64Array::from(vec![1, 2, 3, 4, 5, 6])),
    9260
                Some(NullBuffer::from(vec![true, false, true])),
    9261
            );
    9262
            let converted = ScalarValue::convert_array_to_scalar_vec(&funky).unwrap();
    9263
            assert_eq!(
    9264
                converted,
    9265
                vec![
    9266
                    Some(vec![
    9267
                        ScalarValue::Int64(Some(1)),
    9268
                        ScalarValue::Int64(Some(2))
    9269
                    ]),
    9270
                    None,
    9271
                    Some(vec![ScalarValue::Int64(Some(5))]),
    9272
                ]
    9273
            );
    9274
    9275
            // 4: Offsets + Values looks like this: [[1, 2], [], [5]]
    9276
            // But with NullBuffer it's like this: [[1, 2], NULL, [5]]
    9277
            // The converted result is: [[1, 2], None, [5]]
    9278
            let array4 = ListArray::new(
    9279
                Field::new_list_field(DataType::Int64, true).into(),
    9280
                OffsetBuffer::new(vec![0, 2, 2, 5].into()),
    9281
                Arc::new(Int64Array::from(vec![1, 2, 3, 4, 5, 6])),
    9282
                Some(NullBuffer::from(vec![true, false, true])),
    9283
            );
    9284
            let converted = ScalarValue::convert_array_to_scalar_vec(&array4).unwrap();
    9285
            assert_eq!(
    9286
                converted,
    9287
                vec![
    9288
                    Some(vec![
    9289
                        ScalarValue::Int64(Some(1)),
    9290
                        ScalarValue::Int64(Some(2))
    9291
                    ]),
    9292
                    None,
    9293
                    Some(vec![
    9294
                        ScalarValue::Int64(Some(3)),
    9295
                        ScalarValue::Int64(Some(4)),
    9296
                        ScalarValue::Int64(Some(5)),
    9297
                    ]),
    9298
                ]
    9299
            );
    9300
    9301
            // 5: Offsets + Values looks like this: [[1, 2], [], [5]]
    9302
            // Same as 4, but the middle array is not null, so after conversion it's empty.
    9303
            let array5 = ListArray::new(
    9304
                Field::new_list_field(DataType::Int64, true).into(),
    9305
                OffsetBuffer::new(vec![0, 2, 2, 5].into()),
    9306
                Arc::new(Int64Array::from(vec![1, 2, 3, 4, 5, 6])),
    9307
                Some(NullBuffer::from(vec![true, true, true])),
    9308
            );
    9309
            let converted = ScalarValue::convert_array_to_scalar_vec(&array5).unwrap();
    9310
            assert_eq!(
    9311
                converted,
    9312
                vec![
    9313
                    Some(vec![
    9314
                        ScalarValue::Int64(Some(1)),
    9315
                        ScalarValue::Int64(Some(2))
    9316
                    ]),
    9317
                    Some(vec![]),
    9318
                    Some(vec![
    9319
                        ScalarValue::Int64(Some(3)),
    9320
                        ScalarValue::Int64(Some(4)),
    9321
                        ScalarValue::Int64(Some(5)),
    9322
                    ]),
    9323
                ]
    9324
            );
    9325
        }
    9326
    }