1 // Copyright 2021 Google LLC.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
4 5 package gensupport
6 7 import (
8 "errors"
9 "io"
10 "net"
11 "net/url"
12 "strings"
13 "time"
14 15 "github.com/googleapis/gax-go/v2"
16 "google.golang.org/api/googleapi"
17 )
18 19 // Backoff is an interface around gax.Backoff's Pause method, allowing tests to provide their
20 // own implementation.
21 type Backoff interface {
22 Pause() time.Duration
23 }
24 25 // These are declared as global variables so that tests can overwrite them.
26 var (
27 // Default per-chunk deadline for resumable uploads.
28 defaultRetryDeadline = 32 * time.Second
29 // Default backoff timer.
30 backoff = func() Backoff {
31 return &gax.Backoff{Initial: 100 * time.Millisecond}
32 }
33 )
34 35 const (
36 // statusTooManyRequests is returned by the storage API if the
37 // per-project limits have been temporarily exceeded. The request
38 // should be retried.
39 // https://cloud.google.com/storage/docs/json_api/v1/status-codes#standardcodes
40 statusTooManyRequests = 429
41 42 // statusRequestTimeout is returned by the storage API if the
43 // upload connection was broken. The request should be retried.
44 statusRequestTimeout = 408
45 )
46 47 // shouldRetry indicates whether an error is retryable for the purposes of this
48 // package, unless a ShouldRetry func is specified by the RetryConfig instead.
49 // It follows guidance from
50 // https://cloud.google.com/storage/docs/exponential-backoff .
51 func shouldRetry(status int, err error) bool {
52 if 500 <= status && status <= 599 {
53 return true
54 }
55 if status == statusTooManyRequests || status == statusRequestTimeout {
56 return true
57 }
58 if errors.Is(err, io.ErrUnexpectedEOF) {
59 return true
60 }
61 if errors.Is(err, net.ErrClosed) {
62 return true
63 }
64 switch e := err.(type) {
65 case *net.OpError, *url.Error:
66 // Retry socket-level errors ECONNREFUSED and ECONNRESET (from syscall).
67 // Unfortunately the error type is unexported, so we resort to string
68 // matching.
69 retriable := []string{"connection refused", "connection reset", "broken pipe"}
70 for _, s := range retriable {
71 if strings.Contains(e.Error(), s) {
72 return true
73 }
74 }
75 case interface{ Temporary() bool }:
76 if e.Temporary() {
77 return true
78 }
79 }
80 81 // If error unwrapping is available, use this to examine wrapped
82 // errors.
83 if e, ok := err.(interface{ Unwrap() error }); ok {
84 return shouldRetry(status, e.Unwrap())
85 }
86 return false
87 }
88 89 // RetryConfig allows configuration of backoff timing and retryable errors.
90 type RetryConfig struct {
91 Backoff *gax.Backoff
92 ShouldRetry func(err error) bool
93 }
94 95 // Get a new backoff object based on the configured values.
96 func (r *RetryConfig) backoff() Backoff {
97 if r == nil || r.Backoff == nil {
98 return backoff()
99 }
100 return &gax.Backoff{
101 Initial: r.Backoff.Initial,
102 Max: r.Backoff.Max,
103 Multiplier: r.Backoff.Multiplier,
104 }
105 }
106 107 // This is kind of hacky; it is necessary because ShouldRetry expects to
108 // handle HTTP errors via googleapi.Error, but the error has not yet been
109 // wrapped with a googleapi.Error at this layer, and the ErrorFunc type
110 // in the manual layer does not pass in a status explicitly as it does
111 // here. So, we must wrap error status codes in a googleapi.Error so that
112 // ShouldRetry can parse this correctly.
113 func (r *RetryConfig) errorFunc() func(status int, err error) bool {
114 if r == nil || r.ShouldRetry == nil {
115 return shouldRetry
116 }
117 return func(status int, err error) bool {
118 if status >= 400 {
119 return r.ShouldRetry(&googleapi.Error{Code: status})
120 }
121 return r.ShouldRetry(err)
122 }
123 }
124