chore: perform checksum for single shot uploads (#13733)

Author: krishnamd-jkp

storage/doc.go

@@ -407,9 +407,14 @@ roles which must be enabled in order to do the export successfully. To
 disable this export, you can use the [WithDisabledClientMetrics] client
 option.
 
-The gRPC client automatically computes and sends CRC32C checksums for uploads using [Writer],
-which provides an additional layer of data integrity validation when compared to the HTTP client.
-This behavior can optionally be disabled by using [Writer.DisableAutoChecksum].
+The client automatically computes and sends CRC32C checksums for uploads using [Writer],
+providing an additional layer of data integrity validation with a slight CPU overhead.
+
+Note: With a chunk size of 0 (no buffering) in JSON uploads, an auto-calculated checksum mismatch
+returns an error but may leave corrupt data on the server, requiring manual cleanup. This risk does not
+apply to single-shot uploads when user-provided checksum is provided.
+
+Automatic checksumming can be disabled using [Writer.DisableAutoChecksum].
 
 # Storage Control API
 

storage/go.mod

@@ -6,7 +6,7 @@ retract [v1.25.0, v1.27.0] // due to https://github.com/googleapis/google-cloud-
 
 require (
 	cloud.google.com/go v0.123.0
-	cloud.google.com/go/auth v0.18.0
+	cloud.google.com/go/auth v0.18.1
 	cloud.google.com/go/compute/metadata v0.9.0
 	cloud.google.com/go/iam v1.5.3
 	cloud.google.com/go/longrunning v0.8.0
@@ -22,10 +22,10 @@ require (
 	go.opentelemetry.io/otel/trace v1.39.0
 	golang.org/x/oauth2 v0.34.0
 	golang.org/x/sync v0.19.0
-	google.golang.org/api v0.259.0
+	google.golang.org/api v0.265.0
 	google.golang.org/genproto v0.0.0-20251202230838-ff82c1b0f217
-	google.golang.org/genproto/googleapis/api v0.0.0-20251222181119-0a764e51fe1b
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b
+	google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409
 	google.golang.org/grpc v1.78.0
 	google.golang.org/protobuf v1.36.11
 )
@@ -46,16 +46,16 @@ require (
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/google/martian/v3 v3.3.3 // indirect
 	github.com/google/s2a-go v0.1.9 // indirect
-	github.com/googleapis/enterprise-certificate-proxy v0.3.7 // indirect
+	github.com/googleapis/enterprise-certificate-proxy v0.3.11 // indirect
 	github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect
 	github.com/spiffe/go-spiffe/v2 v2.6.0 // indirect
 	go.opentelemetry.io/auto/sdk v1.2.1 // indirect
 	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 // indirect
 	go.opentelemetry.io/otel/metric v1.39.0 // indirect
-	golang.org/x/crypto v0.46.0 // indirect
-	golang.org/x/net v0.48.0 // indirect
-	golang.org/x/sys v0.39.0 // indirect
-	golang.org/x/text v0.32.0 // indirect
+	golang.org/x/crypto v0.47.0 // indirect
+	golang.org/x/net v0.49.0 // indirect
+	golang.org/x/sys v0.40.0 // indirect
+	golang.org/x/text v0.33.0 // indirect
 	golang.org/x/time v0.14.0 // indirect
 )

storage/go.sum

@@ -2,8 +2,8 @@ cel.dev/expr v0.24.0 h1:56OvJKSH3hDGL0ml5uSxZmz3/3Pq4tJ+fb1unVLAFcY=
 cel.dev/expr v0.24.0/go.mod h1:hLPLo1W4QUmuYdA72RBX06QTs6MXw941piREPl3Yfiw=
 cloud.google.com/go v0.123.0 h1:2NAUJwPR47q+E35uaJeYoNhuNEM9kM8SjgRgdeOJUSE=
 cloud.google.com/go v0.123.0/go.mod h1:xBoMV08QcqUGuPW65Qfm1o9Y4zKZBpGS+7bImXLTAZU=
-cloud.google.com/go/auth v0.18.0 h1:wnqy5hrv7p3k7cShwAU/Br3nzod7fxoqG+k0VZ+/Pk0=
-cloud.google.com/go/auth v0.18.0/go.mod h1:wwkPM1AgE1f2u6dG443MiWoD8C3BtOywNsUMcUTVDRo=
+cloud.google.com/go/auth v0.18.1 h1:IwTEx92GFUo2pJ6Qea0EU3zYvKnTAeRCODxfA/G5UWs=
+cloud.google.com/go/auth v0.18.1/go.mod h1:GfTYoS9G3CWpRA3Va9doKN9mjPGRS+v41jmZAhBzbrA=
 cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc=
 cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c=
 cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs=
@@ -61,8 +61,8 @@ github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0=
 github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/googleapis/enterprise-certificate-proxy v0.3.7 h1:zrn2Ee/nWmHulBx5sAVrGgAa0f2/R35S4DJwfFaUPFQ=
-github.com/googleapis/enterprise-certificate-proxy v0.3.7/go.mod h1:MkHOF77EYAE7qfSuSS9PU6g4Nt4e11cnsDUowfwewLA=
+github.com/googleapis/enterprise-certificate-proxy v0.3.11 h1:vAe81Msw+8tKUxi2Dqh/NZMz7475yUvmRIkXr4oN2ao=
+github.com/googleapis/enterprise-certificate-proxy v0.3.11/go.mod h1:RFV7MUdlb7AgEq2v7FmMCfeSMCllAzWxFgRdusoGks8=
 github.com/googleapis/gax-go/v2 v2.16.0 h1:iHbQmKLLZrexmb0OSsNGTeSTS0HO4YvFOG8g5E4Zd0Y=
 github.com/googleapis/gax-go/v2 v2.16.0/go.mod h1:o1vfQjjNZn4+dPnRdl/4ZD7S9414Y4xA+a/6Icj6l14=
 github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo=
@@ -95,30 +95,30 @@ go.opentelemetry.io/otel/trace v1.39.0 h1:2d2vfpEDmCJ5zVYz7ijaJdOF59xLomrvj7bjt6
 go.opentelemetry.io/otel/trace v1.39.0/go.mod h1:88w4/PnZSazkGzz/w84VHpQafiU4EtqqlVdxWy+rNOA=
 go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
 go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
-golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU=
-golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0=
-golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU=
-golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY=
+golang.org/x/crypto v0.47.0 h1:V6e3FRj+n4dbpw86FJ8Fv7XVOql7TEwpHapKoMJ/GO8=
+golang.org/x/crypto v0.47.0/go.mod h1:ff3Y9VzzKbwSSEzWqJsJVBnWmRwRSHt/6Op5n9bQc4A=
+golang.org/x/net v0.49.0 h1:eeHFmOGUTtaaPSGNmjBKpbng9MulQsJURQUAfUwY++o=
+golang.org/x/net v0.49.0/go.mod h1:/ysNB2EvaqvesRkuLAyjI1ycPZlQHM3q01F02UY/MV8=
 golang.org/x/oauth2 v0.34.0 h1:hqK/t4AKgbqWkdkcAeI8XLmbK+4m4G5YeQRrmiotGlw=
 golang.org/x/oauth2 v0.34.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA=
 golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4=
 golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
-golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk=
-golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
-golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU=
-golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY=
+golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ=
+golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
+golang.org/x/text v0.33.0 h1:B3njUFyqtHDUI5jMn1YIr5B0IE2U0qck04r6d4KPAxE=
+golang.org/x/text v0.33.0/go.mod h1:LuMebE6+rBincTi9+xWTY8TztLzKHc/9C1uBCG27+q8=
 golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI=
 golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4=
 gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk=
 gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E=
-google.golang.org/api v0.259.0 h1:90TaGVIxScrh1Vn/XI2426kRpBqHwWIzVBzJsVZ5XrQ=
-google.golang.org/api v0.259.0/go.mod h1:LC2ISWGWbRoyQVpxGntWwLWN/vLNxxKBK9KuJRI8Te4=
+google.golang.org/api v0.265.0 h1:FZvfUdI8nfmuNrE34aOWFPmLC+qRBEiNm3JdivTvAAU=
+google.golang.org/api v0.265.0/go.mod h1:uAvfEl3SLUj/7n6k+lJutcswVojHPp2Sp08jWCu8hLY=
 google.golang.org/genproto v0.0.0-20251202230838-ff82c1b0f217 h1:GvESR9BIyHUahIb0NcTum6itIWtdoglGX+rnGxm2934=
 google.golang.org/genproto v0.0.0-20251202230838-ff82c1b0f217/go.mod h1:yJ2HH4EHEDTd3JiLmhds6NkJ17ITVYOdV3m3VKOnws0=
-google.golang.org/genproto/googleapis/api v0.0.0-20251222181119-0a764e51fe1b h1:uA40e2M6fYRBf0+8uN5mLlqUtV192iiksiICIBkYJ1E=
-google.golang.org/genproto/googleapis/api v0.0.0-20251222181119-0a764e51fe1b/go.mod h1:Xa7le7qx2vmqB/SzWUBa7KdMjpdpAHlh5QCSnjessQk=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b h1:Mv8VFug0MP9e5vUxfBcE3vUkV6CImK3cMNMIDFjmzxU=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ=
+google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217 h1:fCvbg86sFXwdrl5LgVcTEvNC+2txB5mgROGmRL5mrls=
+google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217/go.mod h1:+rXWjjaukWZun3mLfjmVnQi18E1AsFbDN9QdJ5YXLto=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409 h1:H86B94AW+VfJWDqFeEbBPhEtHzJwJfTbgE2lZa54ZAQ=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ=
 google.golang.org/grpc v1.78.0 h1:K1XZG/yGDJnzMdd/uZHAkVqJE+xIDOcmdSFZkBUicNc=
 google.golang.org/grpc v1.78.0/go.mod h1:I47qjTo4OKbMkjA/aOOwxDIiPSBofUtQUI5EfpWvW7U=
 google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=

storage/http_client.go

@@ -983,11 +983,48 @@ func (c *httpStorageClient) newRangeReaderJSON(ctx context.Context, params *newR
 	return parseReadResponse(res, params, reopen)
 }
 
+// httpInternalWriter writes data for an HTTP upload. For single-shot uploads,
+// it also calculates the CRC32C checksum of the data and validates it against
+// the checksum returned by the server.
 type httpInternalWriter struct {
 	*io.PipeWriter
+	chunkSize          int
+	checksumDisabled   bool
+	fullObjectChecksum uint32
+	// In single-shot mode, the server-provided checksum is received on this
+	// channel for validation after the upload is complete.
+	serverChecksumChan chan uint32
 }
 
-func (hiw httpInternalWriter) Flush() (int64, error) {
+// validateChecksum validates the computed checksum against the server-provided checksum.
+func (hiw *httpInternalWriter) validateChecksumFromServer() error {
+	serverChecksum, ok := <-hiw.serverChecksumChan
+	// Do not check for channel closure as error is already set on the writer
+	// if serverChecksumChan is closed without checksum
+	if ok && hiw.fullObjectChecksum != serverChecksum {
+		return fmt.Errorf("storage: object checksum mismatch: computed %q, server %q; the bucket may contain corrupted object", encodeUint32(hiw.fullObjectChecksum), encodeUint32(serverChecksum))
+	}
+	return nil
+}
+
+func (hiw *httpInternalWriter) Write(data []byte) (n int, err error) {
+	if !hiw.checksumDisabled && hiw.chunkSize == 0 {
+		hiw.fullObjectChecksum = crc32.Update(hiw.fullObjectChecksum, crc32cTable, data)
+	}
+	return hiw.PipeWriter.Write(data)
+}
+
+func (hiw *httpInternalWriter) Close() error {
+	if err := hiw.PipeWriter.Close(); err != nil {
+		return err
+	}
+	if !hiw.checksumDisabled && hiw.chunkSize == 0 {
+		return hiw.validateChecksumFromServer()
+	}
+	return nil
+}
+
+func (hiw *httpInternalWriter) Flush() (int64, error) {
 	return 0, errors.New("Writer.Flush is only supported for gRPC-based clients")
 }
 
@@ -1014,15 +1051,20 @@ func (c *httpStorageClient) OpenWriter(params *openWriterParams, opts ...storage
 	if params.chunkTransferTimeout != 0 {
 		mediaOpts = append(mediaOpts, googleapi.ChunkTransferTimeout(params.chunkTransferTimeout))
 	}
-	if !params.disableAutoChecksum {
-		mediaOpts = append(mediaOpts, googleapi.EnableAutoChecksum())
-	}
 
 	pr, pw := io.Pipe()
-
+	var (
+		serverChecksumChan = make(chan uint32, 1)
+		checksumDisabled   = params.disableAutoChecksum || params.sendCRC32C
+	)
+	if !checksumDisabled {
+		mediaOpts = append(mediaOpts, googleapi.EnableAutoChecksum())
+	}
 	go func() {
-		defer close(params.donec)
-
+		defer func() {
+			close(params.donec)
+			close(serverChecksumChan)
+		}()
 		rawObj := attrs.toRawObject(params.bucket)
 		if params.sendCRC32C {
 			rawObj.Crc32c = encodeUint32(attrs.CRC32C)
@@ -1084,10 +1126,18 @@ func (c *httpStorageClient) OpenWriter(params *openWriterParams, opts ...storage
 			pr.CloseWithError(err)
 			return
 		}
-		setObj(newObject(resp))
+		newObj := newObject(resp)
+		if !checksumDisabled && params.chunkSize == 0 {
+			serverChecksumChan <- newObj.CRC32C
+		}
+		setObj(newObj)
 	}()
-
-	return httpInternalWriter{pw}, nil
+	return &httpInternalWriter{
+		PipeWriter:         pw,
+		chunkSize:          params.chunkSize,
+		serverChecksumChan: serverChecksumChan,
+		checksumDisabled:   checksumDisabled,
+	}, nil
 }
 
 // IAM methods.

storage/http_client_test.go

@@ -16,6 +16,7 @@ package storage
 
 import (
 	"context"
+	"fmt"
 	"net/http"
 	"testing"
 
@@ -125,3 +126,48 @@ func TestAppendableWriteUnsupported(t *testing.T) {
 		t.Errorf("OpenWriter: got ok; want error")
 	}
 }
+
+func TestValidateChecksumFromServer(t *testing.T) {
+	correctChecksum := 1
+	tests := []struct {
+		name          string
+		wrongChecksum bool
+		wantErr       error
+	}{
+		{
+			name:          "correct checksum",
+			wrongChecksum: false,
+		},
+		{
+			name:          "wrong checksum",
+			wrongChecksum: true,
+			wantErr:       fmt.Errorf("storage: computed object checksum (%q) doesn't match with server's object checksum (%q)", encodeUint32(2), encodeUint32(1)),
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			serverChecksumChan := make(chan uint32, 1)
+			checksum := correctChecksum
+			if test.wrongChecksum {
+				checksum = 2
+			}
+			hiw := &httpInternalWriter{
+				serverChecksumChan: serverChecksumChan,
+				fullObjectChecksum: uint32(checksum),
+			}
+			go func() {
+				serverChecksumChan <- uint32(correctChecksum)
+			}()
+			err := hiw.validateChecksumFromServer()
+			if test.wantErr == nil {
+				if err != nil {
+					t.Fatalf("Unexpected error: %v", err)
+				}
+				return
+			}
+			if err.Error() != test.wantErr.Error() {
+				t.Errorf("Want err: %v, got err: %v", err, test.wantErr)
+			}
+		})
+	}
+}

storage/integration_test.go

@@ -1998,6 +1998,11 @@ func TestIntegration_WriterCRC32CValidation(t *testing.T) {
 				chunkSize:  256 * 1024,
 				sendCRC32C: true,
 			},
+			{
+				name:      "small uploads - data size < chunk size with default CRC32C",
+				content:   bytes.Repeat([]byte("a"), 200*1024),
+				chunkSize: 256 * 1024,
+			},
 			{
 				name:      "resumable with default CRC32",
 				content:   bytes.Repeat([]byte("a"), 1*MiB),

storage/writer.go

@@ -47,33 +47,35 @@ type Writer struct {
 	// It is necessary to set this field to true in addition to setting the
 	// Writer's CRC32C field because zero is a valid CRC.
 	//
-	// When using gRPC, the client automatically calculates and sends checksums
-	// per-chunk and for the full object. However, A user-provided checksum takes
-	// precedence over the auto-calculated checksum for full object.
-	// To disable auto checksum behavior, see DisableAutoChecksum.
+	// By default, the client automatically calculates and sends checksums.
+	// When using gRPC, checksums are sent for both individual chunks and the full object.
+	// When using JSON, checksums are sent only for the full object.
+	// However, a user-provided checksum takes precedence over the auto-calculated checksum
+	// for the full object.
 	//
 	// Note: SendCRC32C must be set before the first call to Writer.Write().
 	SendCRC32C bool
 
 	// DisableAutoChecksum disables automatic CRC32C checksum calculation and
-	// validation. By default, the Writer automatically performs checksum
-	// validation for both gRPC and JSON uploads. For gRPC, this includes
-	// validation for both individual chunks and the entire object. For JSON,
-	// checksums are validated for full object but only for resumable uploads. Setting
-	// this to true disables this automatic checksumming behavior.
+	// validation in the Writer. By default, the Writer automatically performs
+	// checksum validation. Setting this to true disables this behavior.
 	//
 	// Disabling automatic checksumming does not prevent a user-provided checksum
 	// from being sent. If SendCRC32C is true and the Writer's CRC32C field is
 	// populated, that checksum will still be sent to GCS for validation.
 	//
+	// For single-shot JSON uploads, a mismatch in the auto-calculated checksum returns
+	// an error but may leave data on the server. This issue does not apply when
+	// user-provided checksum is used. Callers relying on auto-checksum should handle the
+	// error by removing the object or restoring a previous version.
+	//
 	// Automatic CRC32C checksum calculation introduces increased CPU overhead
-	// because of checksum computation in gRPC writes. Use this field to disable
+	// because of checksum computation in writes. Use this field to disable
 	// it if needed.
 	//
 	// Note: DisableAutoChecksum must be set before the first call to
-	// Writer.Write(). Automatic checksumming is not enabled for writes
-	// using the HTTP client or for full object checksums for unfinalized writes to
-	// appendable objects in gRPC.
+	// Writer.Write(). Automatic checksumming is not enabled for full object
+	// checksums for unfinalized writes to appendable objects in gRPC.
 	DisableAutoChecksum bool
 
 	// ChunkSize controls the maximum number of bytes of the object that the