feat(storage): retry on a HTTP 408 response code (#5314)

Author: BrennaEpp

storage/invoke.go

@@ -64,9 +64,9 @@ func shouldRetry(err error) bool {
 	}
 	switch e := err.(type) {
 	case *googleapi.Error:
-		// Retry on 429 and 5xx, according to
+		// Retry on 408, 429, and 5xx, according to
 		// https://cloud.google.com/storage/docs/exponential-backoff.
-		return e.Code == 429 || (e.Code >= 500 && e.Code < 600)
+		return e.Code == 408 || e.Code == 429 || (e.Code >= 500 && e.Code < 600)
 	case *url.Error:
 		// Retry socket-level errors ECONNREFUSED and ENETUNREACH (from syscall).
 		// Unfortunately the error type is unexported, so we resort to string

storage/storage.go

@@ -1900,7 +1900,7 @@ func (ws *withPolicy) apply(config *retryConfig) {
 // By default, the following errors are retried (see invoke.go for the default
 // shouldRetry function):
 //
-// - HTTP responses with codes 429, 502, 503, and 504.
+// - HTTP responses with codes 408, 429, 502, 503, and 504.
 //
 // - Transient network errors such as connection reset and io.ErrUnexpectedEOF.
 //