## Coverage Report

#### Created: 2025-11-22 06:00

next uncovered line (L), next uncovered region (R), next uncovered branch (B)

    /Users/andrewlamb/Software/datafusion/datafusion/physical-expr/src/expressions/column.rs

    Line
    Count
    Source
    1
    // Licensed to the Apache Software Foundation (ASF) under one
    2
    // or more contributor license agreements.  See the NOTICE file
    3
    // distributed with this work for additional information
    4
    // regarding copyright ownership.  The ASF licenses this file
    5
    // to you under the Apache License, Version 2.0 (the
    6
    // "License"); you may not use this file except in compliance
    7
    // with the License.  You may obtain a copy of the License at
    8
    //
    9
    //   http://www.apache.org/licenses/LICENSE-2.0
    10
    //
    11
    // Unless required by applicable law or agreed to in writing,
    12
    // software distributed under the License is distributed on an
    13
    // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
    14
    // KIND, either express or implied.  See the License for the
    15
    // specific language governing permissions and limitations
    16
    // under the License.
    17
    18
    //! Physical column reference: [`Column`]
    19
    20
    use std::any::Any;
    21
    use std::hash::Hash;
    22
    use std::sync::Arc;
    23
    24
    use crate::physical_expr::PhysicalExpr;
    25
    use arrow::datatypes::FieldRef;
    26
    use arrow::{
    27
        datatypes::{DataType, Schema, SchemaRef},
    28
        record_batch::RecordBatch,
    29
    };
    30
    use datafusion_common::tree_node::{Transformed, TreeNode};
    31
    use datafusion_common::{internal_err, plan_err, Result};
    32
    use datafusion_expr::ColumnarValue;
    33
    34
    /// Represents the column at a given index in a RecordBatch
    35
    ///
    36
    /// This is a physical expression that represents a column at a given index in an
    37
    /// arrow [`Schema`] / [`RecordBatch`].
    38
    ///
    39
    /// Unlike the [logical `Expr::Column`], this expression is always resolved by schema index,
    40
    /// even though it does have a name. This is because the physical plan is always
    41
    /// resolved to a specific schema and there is no concept of "relation"
    42
    ///
    43
    /// # Example:
    44
    ///  If the schema is `a`, `b`, `c` the `Column` for `b` would be represented by
    45
    ///  index 1, since `b` is the second column in the schema.
    46
    ///
    47
    /// ```
    48
    /// # use datafusion_physical_expr::expressions::Column;
    49
    /// # use arrow::datatypes::{DataType, Field, Schema};
    50
    /// // Schema with columns a, b, c
    51
    /// let schema = Schema::new(vec![
    52
    ///     Field::new("a", DataType::Int32, false),
    53
    ///     Field::new("b", DataType::Int32, false),
    54
    ///     Field::new("c", DataType::Int32, false),
    55
    /// ]);
    56
    ///
    57
    /// // reference to column b is index 1
    58
    /// let column_b = Column::new_with_schema("b", &schema).unwrap();
    59
    /// assert_eq!(column_b.index(), 1);
    60
    ///
    61
    /// // reference to column c is index 2
    62
    /// let column_c = Column::new_with_schema("c", &schema).unwrap();
    63
    /// assert_eq!(column_c.index(), 2);
    64
    /// ```
    65
    /// [logical `Expr::Column`]: https://docs.rs/datafusion/latest/datafusion/logical_expr/enum.Expr.html#variant.Column
    66
    #[derive(Debug, Hash, PartialEq, Eq, Clone)]
    67
    pub struct Column {
    68
        /// The name of the column (used for debugging and display purposes)
    69
        name: String,
    70
        /// The index of the column in its schema
    71
        index: usize,
    72
    }
    73
    74
    impl Column {
    75
        /// Create a new column expression which references the
    76
        /// column with the given index in the schema.
    77
    47
        pub fn new(name: &str, index: usize) -> Self {
    78
    47
            Self {
    79
    47
                name: name.to_owned(),
    80
    47
                index,
    81
    47
            }
    82
    47
        }
    83
    84
        /// Create a new column expression which references the
    85
        /// column with the given name in the schema
    86
    46
        pub fn new_with_schema(name: &str, schema: &Schema) -> Result<Self> {
    87
    46
            Ok(Column::new(name, schema.index_of(name)?0))
    88
    46
        }
    89
    90
        /// Get the column's name
    91
    2
        pub fn name(&self) -> &str {
    92
    2
            &self.name
    93
    2
        }
    94
    95
        /// Get the column's schema index
    96
    0
        pub fn index(&self) -> usize {
    97
    0
            self.index
    98
    0
        }
    99
    }
    100
    101
    impl std::fmt::Display for Column {
    102
    5
        fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
    103
    5
            write!(f, "{}@{}", self.name, self.index)
    104
    5
        }
    105
    }
    106
    107
    impl PhysicalExpr for Column {
    108
        /// Return a reference to Any that can be used for downcasting
    109
    2
        fn as_any(&self) -> &dyn Any {
    110
    2
            self
    111
    2
        }
    112
    113
        /// Get the data type of this expression, given the schema of the input
    114
    364
        fn data_type(&self, input_schema: &Schema) -> Result<DataType> {
    115
    364
            self.bounds_check(input_schema)?0;
    116
    364
            Ok(input_schema.field(self.index).data_type().clone())
    117
    364
        }
    118
    119
        /// Decide whether this expression is nullable, given the schema of the input
    120
    10
        fn nullable(&self, input_schema: &Schema) -> Result<bool> {
    121
    10
            self.bounds_check(input_schema)?0;
    122
    10
            Ok(input_schema.field(self.index).is_nullable())
    123
    10
        }
    124
    125
        /// Evaluate the expression
    126
    152
        fn evaluate(&self, batch: &RecordBatch) -> Result<ColumnarValue> {
    127
    152
            self.bounds_check(batch.schema().as_ref())?0;
    128
    152
            Ok(ColumnarValue::Array(Arc::clone(batch.column(self.index))))
    129
    152
        }
    130
    131
    0
        fn return_field(&self, input_schema: &Schema) -> Result<FieldRef> {
    132
    0
            Ok(input_schema.field(self.index).clone().into())
    133
    0
        }
    134
    135
    1
        fn children(&self) -> Vec<&Arc<dyn PhysicalExpr>> {
    136
    1
            vec![]
    137
    1
        }
    138
    139
    0
        fn with_new_children(
    140
    0
            self: Arc<Self>,
    141
    0
            _children: Vec<Arc<dyn PhysicalExpr>>,
    142
    0
        ) -> Result<Arc<dyn PhysicalExpr>> {
    143
    0
            Ok(self)
    144
    0
        }
    145
    146
    4
        fn fmt_sql(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
    147
    4
            write!(f, "{}", self.name)
    148
    4
        }
    149
    }
    150
    151
    impl Column {
    152
    526
        fn bounds_check(&self, input_schema: &Schema) -> Result<()> {
    153
    526
            if self.index < input_schema.fields.len() {
    154
    526
                Ok(())
    155
            } else {
    156
    0
                internal_err!(
    157
    0
                    "PhysicalExpr Column references column '{}' at index {} (zero-based) but input schema only has {} columns: {:?}",
    158
                    self.name,
    159
                    self.index,
    160
    0
                    input_schema.fields.len(),
    161
    0
                    input_schema.fields().iter().map(|f| f.name()).collect::<Vec<_>>()
    162
                )
    163
            }
    164
    526
        }
    165
    }
    166
    167
    /// Create a column expression
    168
    44
    pub fn col(name: &str, schema: &Schema) -> Result<Arc<dyn PhysicalExpr>> {
    169
    44
        Ok(Arc::new(Column::new_with_schema(name, schema)?0))
    170
    44
    }
    171
    172
    /// Rewrites an expression according to new schema; i.e. changes the columns it
    173
    /// refers to with the column at corresponding index in the new schema. Returns
    174
    /// an error if the given schema has fewer columns than the original schema.
    175
    /// Note that the resulting expression may not be valid if data types in the
    176
    /// new schema is incompatible with expression nodes.
    177
    0
    pub fn with_new_schema(
    178
    0
        expr: Arc<dyn PhysicalExpr>,
    179
    0
        schema: &SchemaRef,
    180
    0
    ) -> Result<Arc<dyn PhysicalExpr>> {
    181
    0
        Ok(expr
    182
    0
            .transform_up(|expr| {
    183
    0
                if let Some(col) = expr.as_any().downcast_ref::<Column>() {
    184
    0
                    let idx = col.index();
    185
    0
                    let Some(field) = schema.fields().get(idx) else {
    186
    0
                        return plan_err!(
    187
    0
                            "New schema has fewer columns than original schema"
    188
                        );
    189
                    };
    190
    0
                    let new_col = Column::new(field.name(), idx);
    191
    0
                    Ok(Transformed::yes(Arc::new(new_col) as _))
    192
                } else {
    193
    0
                    Ok(Transformed::no(expr))
    194
                }
    195
    0
            })?
    196
            .data)
    197
    0
    }
    198
    199
    #[cfg(test)]
    200
    mod test {
    201
        use super::Column;
    202
        use crate::physical_expr::PhysicalExpr;
    203
    204
        use arrow::array::StringArray;
    205
        use arrow::datatypes::{DataType, Field, Schema};
    206
        use arrow::record_batch::RecordBatch;
    207
    208
        use std::sync::Arc;
    209
    210
        #[test]
    211
    0
        fn out_of_bounds_data_type() {
    212
    0
            let schema = Schema::new(vec![Field::new("foo", DataType::Utf8, true)]);
    213
    0
            let col = Column::new("id", 9);
    214
    0
            let error = col.data_type(&schema).expect_err("error").strip_backtrace();
    215
    0
            assert!("Internal error: PhysicalExpr Column references column 'id' at index 9 (zero-based) \
    216
    0
                 but input schema only has 1 columns: [\"foo\"].\nThis issue was likely caused by a bug \
    217
    0
                 in DataFusion's code. Please help us to resolve this by filing a bug report \
    218
    0
                 in our issue tracker: https://github.com/apache/datafusion/issues".starts_with(&error))
    219
    0
        }
    220
    221
        #[test]
    222
    0
        fn out_of_bounds_nullable() {
    223
    0
            let schema = Schema::new(vec![Field::new("foo", DataType::Utf8, true)]);
    224
    0
            let col = Column::new("id", 9);
    225
    0
            let error = col.nullable(&schema).expect_err("error").strip_backtrace();
    226
    0
            assert!("Internal error: PhysicalExpr Column references column 'id' at index 9 (zero-based) \
    227
    0
                 but input schema only has 1 columns: [\"foo\"].\nThis issue was likely caused by a bug \
    228
    0
                 in DataFusion's code. Please help us to resolve this by filing a bug report \
    229
    0
                 in our issue tracker: https://github.com/apache/datafusion/issues".starts_with(&error));
    230
    0
        }
    231
    232
        #[test]
    233
    0
        fn out_of_bounds_evaluate() {
    234
    0
            let schema = Schema::new(vec![Field::new("foo", DataType::Utf8, true)]);
    235
    0
            let data: StringArray = vec!["data"].into();
    236
    0
            let batch = RecordBatch::try_new(Arc::new(schema), vec![Arc::new(data)]).unwrap();
    237
    0
            let col = Column::new("id", 9);
    238
    0
            let error = col.evaluate(&batch).expect_err("error").strip_backtrace();
    239
    0
            assert!("Internal error: PhysicalExpr Column references column 'id' at index 9 (zero-based) \
    240
    0
                 but input schema only has 1 columns: [\"foo\"].\nThis issue was likely caused by a bug \
    241
    0
                 in DataFusion's code. Please help us to resolve this by filing a bug report \
    242
    0
                 in our issue tracker: https://github.com/apache/datafusion/issues".starts_with(&error));
    243
    0
        }
    244
    }