Skip to content

Commit 8be5e69

Browse files
kris-gaudelkris-gaudel
authored
feat(arrow/_examples): enhance library examples (#394)
### Rationale for this change Enhancing the library examples ### What changes are included in this PR? 3 new examples on how to use the `arrow-go` library: - Reading from a CSV file - Type conversions - Managing performance ### Are these changes tested? Not applicable ### Are there any user-facing changes? No, just enhancing examples
1 parent 14eb8f3 commit 8be5e69

6 files changed

Lines changed: 431 additions & 9 deletions

File tree

‎arrow/array/example_type_conversion_test.go‎

Lines changed: 147 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,147 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing, software
12+
// distributed under the License is distributed on an "AS IS" BASIS,
13+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
// See the License for the specific language governing permissions and
15+
// limitations under the License.
16+
17+
package array_test
18+
19+
import (
20+
"fmt"
21+
22+
"github.com/apache/arrow-go/v18/arrow"
23+
"github.com/apache/arrow-go/v18/arrow/array"
24+
"github.com/apache/arrow-go/v18/arrow/memory"
25+
)
26+
27+
func Example_typeConversion() {
28+
pool := memory.NewGoAllocator()
29+
30+
// 1. Basic type conversion (Int32 to Int64)
31+
fmt.Println("Example 1: Converting Int32 to Int64")
32+
int32Builder := array.NewInt32Builder(pool)
33+
defer int32Builder.Release()
34+
35+
int32Builder.AppendValues([]int32{1, 2, 3, 4, 5}, nil)
36+
int32Array := int32Builder.NewInt32Array()
37+
defer int32Array.Release()
38+
39+
// Convert to Int64
40+
int64Builder := array.NewInt64Builder(pool)
41+
defer int64Builder.Release()
42+
43+
for i := 0; i < int32Array.Len(); i++ {
44+
int64Builder.Append(int64(int32Array.Value(i)))
45+
}
46+
int64Array := int64Builder.NewInt64Array()
47+
defer int64Array.Release()
48+
49+
fmt.Printf("Original Int32 values: %v\n", int32Array.Int32Values())
50+
fmt.Printf("Converted Int64 values: %v\n", int64Array.Int64Values())
51+
52+
// 2. Handling nullable fields
53+
fmt.Println("\nExample 2: Working with nullable fields")
54+
float64Builder := array.NewFloat64Builder(pool)
55+
defer float64Builder.Release()
56+
57+
values := []float64{1.1, 2.2, 3.3, 4.4, 5.5}
58+
valid := []bool{true, true, false, true, false}
59+
float64Builder.AppendValues(values, valid)
60+
float64Array := float64Builder.NewFloat64Array()
61+
defer float64Array.Release()
62+
63+
stringBuilder := array.NewStringBuilder(pool)
64+
defer stringBuilder.Release()
65+
66+
for i := 0; i < float64Array.Len(); i++ {
67+
if float64Array.IsNull(i) {
68+
stringBuilder.AppendNull()
69+
} else {
70+
stringBuilder.Append(fmt.Sprintf("%.2f", float64Array.Value(i)))
71+
}
72+
}
73+
stringArray := stringBuilder.NewStringArray()
74+
defer stringArray.Release()
75+
76+
fmt.Println("Original Float64 values (with nulls):")
77+
for i := 0; i < float64Array.Len(); i++ {
78+
if float64Array.IsNull(i) {
79+
fmt.Printf(" [%d]: null\n", i)
80+
} else {
81+
fmt.Printf(" [%d]: %.2f\n", i, float64Array.Value(i))
82+
}
83+
}
84+
85+
fmt.Println("\nConverted String values (with nulls):")
86+
for i := 0; i < stringArray.Len(); i++ {
87+
if stringArray.IsNull(i) {
88+
fmt.Printf(" [%d]: null\n", i)
89+
} else {
90+
fmt.Printf(" [%d]: %s\n", i, stringArray.Value(i))
91+
}
92+
}
93+
94+
// 3. Working with nested types (List)
95+
fmt.Println("\nExample 3: Working with nested types (List)")
96+
listBuilder := array.NewListBuilder(pool, arrow.PrimitiveTypes.Int32)
97+
defer listBuilder.Release()
98+
99+
valueBuilder := listBuilder.ValueBuilder().(*array.Int32Builder)
100+
101+
listBuilder.Append(true)
102+
valueBuilder.AppendValues([]int32{1, 2}, nil)
103+
104+
listBuilder.Append(true)
105+
valueBuilder.AppendValues([]int32{3, 4, 5}, nil)
106+
107+
listBuilder.Append(true)
108+
valueBuilder.AppendValues([]int32{6}, nil)
109+
110+
listArray := listBuilder.NewListArray()
111+
defer listArray.Release()
112+
113+
// Convert list to string representation
114+
fmt.Println("List of lists:")
115+
for i := 0; i < listArray.Len(); i++ {
116+
values := listArray.ListValues().(*array.Int32).Int32Values()
117+
offset := listArray.Offsets()[i]
118+
length := listArray.Offsets()[i+1] - offset
119+
fmt.Printf(" List %d: %v\n", i, values[offset:offset+length])
120+
}
121+
122+
// Output:
123+
// Example 1: Converting Int32 to Int64
124+
// Original Int32 values: [1 2 3 4 5]
125+
// Converted Int64 values: [1 2 3 4 5]
126+
//
127+
// Example 2: Working with nullable fields
128+
// Original Float64 values (with nulls):
129+
// [0]: 1.10
130+
// [1]: 2.20
131+
// [2]: null
132+
// [3]: 4.40
133+
// [4]: null
134+
//
135+
// Converted String values (with nulls):
136+
// [0]: 1.10
137+
// [1]: 2.20
138+
// [2]: null
139+
// [3]: 4.40
140+
// [4]: null
141+
//
142+
// Example 3: Working with nested types (List)
143+
// List of lists:
144+
// List 0: [1 2]
145+
// List 1: [3 4 5]
146+
// List 2: [6]
147+
}

‎arrow/csv/example_csv_test.go‎

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package csv_test
19+
20+
import (
21+
"fmt"
22+
"os"
23+
24+
"github.com/apache/arrow-go/v18/arrow"
25+
"github.com/apache/arrow-go/v18/arrow/array"
26+
arrowcsv "github.com/apache/arrow-go/v18/arrow/csv"
27+
)
28+
29+
func Example_reader() {
30+
filePath := "../../arrow-testing/data/csv/aggregate_test_100.csv" // Test csv file
31+
f, err := os.Open(filePath)
32+
if err != nil {
33+
fmt.Printf("Failed to open file: %v\n", err)
34+
return
35+
}
36+
defer f.Close()
37+
38+
// Schema defined in the csv file
39+
schema := arrow.NewSchema([]arrow.Field{
40+
{Name: "c1", Type: arrow.BinaryTypes.String, Nullable: true},
41+
{Name: "c2", Type: arrow.PrimitiveTypes.Int64, Nullable: true},
42+
{Name: "c3", Type: arrow.PrimitiveTypes.Int64, Nullable: true},
43+
{Name: "c4", Type: arrow.PrimitiveTypes.Int64, Nullable: true},
44+
{Name: "c5", Type: arrow.PrimitiveTypes.Int64, Nullable: true},
45+
{Name: "c6", Type: arrow.PrimitiveTypes.Int64, Nullable: true},
46+
{Name: "c7", Type: arrow.PrimitiveTypes.Int64, Nullable: true},
47+
{Name: "c8", Type: arrow.PrimitiveTypes.Int64, Nullable: true},
48+
{Name: "c9", Type: arrow.PrimitiveTypes.Int64, Nullable: true},
49+
{Name: "c10", Type: arrow.PrimitiveTypes.Int64, Nullable: true},
50+
{Name: "c11", Type: arrow.PrimitiveTypes.Float64, Nullable: true},
51+
{Name: "c12", Type: arrow.PrimitiveTypes.Float64, Nullable: true},
52+
{Name: "c13", Type: arrow.BinaryTypes.String, Nullable: true},
53+
}, nil)
54+
55+
reader := arrowcsv.NewReader(f, schema, arrowcsv.WithHeader(true), arrowcsv.WithChunk(-1))
56+
defer reader.Release()
57+
58+
// Read the first record
59+
if !reader.Next() {
60+
if err := reader.Err(); err != nil {
61+
fmt.Printf("Error reading CSV: %v\n", err)
62+
return
63+
}
64+
fmt.Println("No records found")
65+
return
66+
}
67+
68+
// Get the record but don't release it - the reader will handle that
69+
record := reader.Record()
70+
71+
fmt.Printf("Number of rows: %d\n", record.NumRows())
72+
fmt.Printf("Number of columns: %d\n", record.NumCols())
73+
fmt.Println()
74+
75+
fmt.Println("Basic statistics for numeric columns:")
76+
for i := 1; i < 10; i++ { // cols c2 through c10 are Int64
77+
col := record.Column(i).(*array.Int64)
78+
var sum int64
79+
for j := 0; j < col.Len(); j++ {
80+
sum += col.Value(j)
81+
}
82+
avg := float64(sum) / float64(col.Len())
83+
fmt.Printf("Column c%d: Average = %.2f\n", i+1, avg)
84+
}
85+
86+
for i := 10; i < 12; i++ { // cols c11 and c12 are Float64
87+
col := record.Column(i).(*array.Float64)
88+
var sum float64
89+
for j := 0; j < col.Len(); j++ {
90+
sum += col.Value(j)
91+
}
92+
avg := sum / float64(col.Len())
93+
fmt.Printf("Column c%d: Average = %.4f\n", i+1, avg)
94+
}
95+
96+
// Output:
97+
// Number of rows: 100
98+
// Number of columns: 13
99+
//
100+
// Basic statistics for numeric columns:
101+
// Column c2: Average = 2.85
102+
// Column c3: Average = 7.81
103+
// Column c4: Average = 2319.97
104+
// Column c5: Average = 158626279.61
105+
// Column c6: Average = 59276376114661656.00
106+
// Column c7: Average = 130.60
107+
// Column c8: Average = 30176.41
108+
// Column c9: Average = 2220897700.60
109+
// Column c10: Average = -86834033398685392.00
110+
// Column c11: Average = 0.4793
111+
// Column c12: Average = 0.5090
112+
}

‎arrow/example_table_creation_test.go‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package arrow_test
19+
20+
import (
21+
"fmt"
22+
23+
"github.com/apache/arrow-go/v18/arrow"
24+
"github.com/apache/arrow-go/v18/arrow/array"
25+
"github.com/apache/arrow-go/v18/arrow/math"
26+
"github.com/apache/arrow-go/v18/arrow/memory"
27+
)
28+
29+
func Example_tableCreation() {
30+
// Create a schema with three fields
31+
schema := arrow.NewSchema([]arrow.Field{
32+
{Name: "intField", Type: arrow.PrimitiveTypes.Int64, Nullable: false},
33+
{Name: "stringField", Type: arrow.BinaryTypes.String, Nullable: false},
34+
{Name: "floatField", Type: arrow.PrimitiveTypes.Float64, Nullable: true},
35+
}, nil)
36+
37+
// Create a record builder
38+
builder := array.NewRecordBuilder(memory.DefaultAllocator, schema)
39+
defer builder.Release()
40+
41+
// Append values to each field
42+
builder.Field(0).(*array.Int64Builder).AppendValues([]int64{1, 2, 3, 4, 5}, nil)
43+
builder.Field(1).(*array.StringBuilder).AppendValues([]string{"a", "b", "c", "d", "e"}, nil)
44+
builder.Field(2).(*array.Float64Builder).AppendValues([]float64{1, 0, 3, 0, 5}, []bool{true, false, true, false, true})
45+
46+
// Create a record
47+
rec := builder.NewRecord()
48+
defer rec.Release()
49+
50+
// Create a table from the record
51+
tbl := array.NewTableFromRecords(schema, []arrow.Record{rec})
52+
defer tbl.Release()
53+
54+
// Calculate sum of floatField
55+
sum := math.Float64.Sum(tbl.Column(2).Data().Chunk(0).(*array.Float64))
56+
fmt.Printf("Sum of floatField: %v\n", sum)
57+
58+
// Print the table contents
59+
fmt.Println("\nTable contents:")
60+
fmt.Printf("Number of rows: %d\n", tbl.NumRows())
61+
fmt.Printf("Number of columns: %d\n", tbl.NumCols())
62+
fmt.Println("\nColumn names:")
63+
for i := 0; i < int(tbl.NumCols()); i++ {
64+
fmt.Printf(" %s\n", tbl.Column(i).Name())
65+
}
66+
67+
// Output:
68+
// Sum of floatField: 9
69+
//
70+
// Table contents:
71+
// Number of rows: 5
72+
// Number of columns: 3
73+
//
74+
// Column names:
75+
// intField
76+
// stringField
77+
// floatField
78+
}

0 commit comments

Comments
 (0)
⚡