fix(table): seed UpdateSchema id counter from metadata LastColumnID (#936)

## Problem

`NewUpdateSchema` seeds its fresh-id counter from
`txn.meta.CurrentSchema().HighestFieldID()`. The Iceberg spec reserves
`last-column-id` on table metadata as the monotonic counter for this
purpose; Java's
[`SchemaUpdate`](https://github.com/apache/iceberg/blob/main/core/src/main/java/org/apache/iceberg/SchemaUpdate.java)
seeds from `metadata.lastColumnId()`.

When the current schema's highest id is lower than the lifetime max —
after a previous evolution added columns that were later dropped, or
after a schema swap — `AddColumn` allocates ids already referenced by
fields in historical schemas. Catalogs that index the full schema
history (e.g. AWS Glue's `schemasToGlueColumns`) then reject the commit
with a duplicate field id error.

## Fix

Seed `lastColumnID` from `txn.meta.LastColumnID()` in `NewUpdateSchema`.
Adds a public `LastColumnID()` getter on `*MetadataBuilder` — the
`Metadata` interface already exposes this value, but `txn.meta` is the
builder.

`MetadataBuilder.AddSchema` already keeps `last-column-id` monotonic via
`max(b.lastColumnId, schema.HighestFieldID())`, so no other allocator
sites need changes.

## Testing

Adds `TestAddColumnMonotonicFieldIDs`: constructs metadata whose
`last-column-id` exceeds the current schema's `HighestFieldID()` (by
adding a higher-id schema while leaving the original as current) and
asserts that `AddColumn` allocates above `last-column-id`. Confirmed
failing on `main` before the fix and passing after.

`go test ./...` passes.

Fixes #935

Signed-off-by: Ashley Jeffs <ash@jeffail.uk>

Author: Jeffail

table/metadata.go

@@ -296,6 +296,10 @@ func (b *MetadataBuilder) CurrentSchema() *iceberg.Schema {
 
 func (b *MetadataBuilder) LastUpdatedMS() int64 { return b.lastUpdatedMS }
 
+// LastColumnID returns the highest field id ever assigned in this table's
+// lifetime, as tracked by the Iceberg spec's last-column-id counter.
+func (b *MetadataBuilder) LastColumnID() int { return b.lastColumnId }
+
 func (b *MetadataBuilder) nextSequenceNumber() int64 {
 	if b.formatVersion > 1 {
 		if b.lastSequenceNumber == nil {

table/update_schema.go

@@ -133,9 +133,14 @@ func WithNameMapping(nameMapping iceberg.NameMapping) UpdateSchemaOption {
 // Returns an UpdateSchema instance that can be used to build and apply schema changes.
 func NewUpdateSchema(txn *Transaction, caseSensitive bool, allowIncompatibleChanges bool, opts ...UpdateSchemaOption) *UpdateSchema {
 	u := &UpdateSchema{
-		txn:          txn,
-		schema:       nil,
-		lastColumnID: txn.meta.CurrentSchema().HighestFieldID(),
+		txn:    txn,
+		schema: nil,
+		// Seed from metadata's last-column-id rather than the current schema's
+		// highest field id. Per the Iceberg spec, last-column-id is a monotonic
+		// counter that preserves ids across schema evolution (including
+		// deletions) so that newly-allocated ids never collide with ids still
+		// referenced by historical schemas.
+		lastColumnID: txn.meta.LastColumnID(),
 
 		deletes: make(map[int]struct{}),
 		updates: make(map[int]map[int]iceberg.NestedField),

table/update_schema_test.go

@@ -913,3 +913,51 @@ func TestBuildUpdates(t *testing.T) {
 		assert.Equal(t, 0, updates[0].(*setCurrentSchemaUpdate).SchemaID)
 	})
 }
+
+// TestAddColumnMonotonicFieldIDs exercises the case where the table's
+// last-column-id is greater than the current schema's highest field id — for
+// example because a previous schema was added that introduced higher ids, or
+// because the highest-id columns were later dropped. The Iceberg spec requires
+// new field ids to be allocated above last-column-id (never to be reused), so
+// AddColumn must seed its id counter from metadata.LastColumnID() rather than
+// the current schema's HighestFieldID().
+func TestAddColumnMonotonicFieldIDs(t *testing.T) {
+	// Start from originalSchema (field ids 1..11, schema id 1).
+	baseMeta, err := NewMetadata(originalSchema, iceberg.UnpartitionedSpec, UnsortedSortOrder, "", nil)
+	assert.NoError(t, err)
+
+	// Add a second schema that introduces higher field ids. This bumps the
+	// metadata's last-column-id to 13 while the current schema is still the
+	// original (highest field id 11).
+	expanded := iceberg.NewSchema(0,
+		iceberg.NestedField{ID: 1, Name: "id", Type: iceberg.PrimitiveTypes.Int32, Required: true},
+		iceberg.NestedField{ID: 12, Name: "extra_a", Type: iceberg.PrimitiveTypes.String, Required: false},
+		iceberg.NestedField{ID: 13, Name: "extra_b", Type: iceberg.PrimitiveTypes.String, Required: false},
+	)
+
+	builder, err := MetadataBuilderFromBase(baseMeta, "")
+	assert.NoError(t, err)
+	assert.NoError(t, builder.AddSchema(expanded))
+
+	meta, err := builder.Build()
+	assert.NoError(t, err)
+
+	assert.Equal(t, 11, meta.CurrentSchema().HighestFieldID(),
+		"precondition: current schema should still be the original with highest id 11")
+	assert.Equal(t, 13, meta.LastColumnID(),
+		"precondition: last-column-id should have been bumped by the expanded schema")
+
+	tbl := New([]string{"id"}, meta, "", nil, nil)
+	txn := tbl.NewTransaction()
+
+	newSchema, err := NewUpdateSchema(txn, true, true).
+		AddColumn([]string{"fresh"}, iceberg.PrimitiveTypes.String, "", false, nil).
+		Apply()
+	assert.NoError(t, err)
+	assert.NotNil(t, newSchema)
+
+	fresh, ok := newSchema.FindFieldByName("fresh")
+	assert.True(t, ok, "new field should be present in the resulting schema")
+	assert.Equal(t, 14, fresh.ID,
+		"new field id must be allocated above metadata.LastColumnID() (13), not reused from the current schema's highest id (11)")
+}