1 // Copyright 2016, Google Inc.
2 // All rights reserved.
3 //
4 // Redistribution and use in source and binary forms, with or without
5 // modification, are permitted provided that the following conditions are
6 // met:
7 //
8 // * Redistributions of source code must retain the above copyright
9 // notice, this list of conditions and the following disclaimer.
10 // * Redistributions in binary form must reproduce the above
11 // copyright notice, this list of conditions and the following disclaimer
12 // in the documentation and/or other materials provided with the
13 // distribution.
14 // * Neither the name of Google Inc. nor the names of its
15 // contributors may be used to endorse or promote products derived from
16 // this software without specific prior written permission.
17 //
18 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29 30 package gax
31 32 import (
33 "errors"
34 "math/rand"
35 "time"
36 37 "google.golang.org/api/googleapi"
38 "google.golang.org/grpc"
39 "google.golang.org/grpc/codes"
40 "google.golang.org/grpc/status"
41 )
42 43 // CallOption is an option used by Invoke to control behaviors of RPC calls.
44 // CallOption works by modifying relevant fields of CallSettings.
45 type CallOption interface {
46 // Resolve applies the option by modifying cs.
47 Resolve(cs *CallSettings)
48 }
49 50 // Retryer is used by Invoke to determine retry behavior.
51 type Retryer interface {
52 // Retry reports whether a request should be retried and how long to pause before retrying
53 // if the previous attempt returned with err. Invoke never calls Retry with nil error.
54 Retry(err error) (pause time.Duration, shouldRetry bool)
55 }
56 57 type retryerOption func() Retryer
58 59 func (o retryerOption) Resolve(s *CallSettings) {
60 s.Retry = o
61 }
62 63 // WithRetry sets CallSettings.Retry to fn.
64 func WithRetry(fn func() Retryer) CallOption {
65 return retryerOption(fn)
66 }
67 68 // OnErrorFunc returns a Retryer that retries if and only if the previous attempt
69 // returns an error that satisfies shouldRetry.
70 //
71 // Pause times between retries are specified by bo. bo is only used for its
72 // parameters; each Retryer has its own copy.
73 func OnErrorFunc(bo Backoff, shouldRetry func(err error) bool) Retryer {
74 return &errorRetryer{
75 shouldRetry: shouldRetry,
76 backoff: bo,
77 }
78 }
79 80 type errorRetryer struct {
81 backoff Backoff
82 shouldRetry func(err error) bool
83 }
84 85 func (r *errorRetryer) Retry(err error) (time.Duration, bool) {
86 if r.shouldRetry(err) {
87 return r.backoff.Pause(), true
88 }
89 90 return 0, false
91 }
92 93 // OnCodes returns a Retryer that retries if and only if
94 // the previous attempt returns a GRPC error whose error code is stored in cc.
95 // Pause times between retries are specified by bo.
96 //
97 // bo is only used for its parameters; each Retryer has its own copy.
98 func OnCodes(cc []codes.Code, bo Backoff) Retryer {
99 return &boRetryer{
100 backoff: bo,
101 codes: append([]codes.Code(nil), cc...),
102 }
103 }
104 105 type boRetryer struct {
106 backoff Backoff
107 codes []codes.Code
108 }
109 110 func (r *boRetryer) Retry(err error) (time.Duration, bool) {
111 st, ok := status.FromError(err)
112 if !ok {
113 return 0, false
114 }
115 c := st.Code()
116 for _, rc := range r.codes {
117 if c == rc {
118 return r.backoff.Pause(), true
119 }
120 }
121 return 0, false
122 }
123 124 // OnHTTPCodes returns a Retryer that retries if and only if
125 // the previous attempt returns a googleapi.Error whose status code is stored in
126 // cc. Pause times between retries are specified by bo.
127 //
128 // bo is only used for its parameters; each Retryer has its own copy.
129 func OnHTTPCodes(bo Backoff, cc ...int) Retryer {
130 codes := make(map[int]bool, len(cc))
131 for _, c := range cc {
132 codes[c] = true
133 }
134 135 return &httpRetryer{
136 backoff: bo,
137 codes: codes,
138 }
139 }
140 141 type httpRetryer struct {
142 backoff Backoff
143 codes map[int]bool
144 }
145 146 func (r *httpRetryer) Retry(err error) (time.Duration, bool) {
147 var gerr *googleapi.Error
148 if !errors.As(err, &gerr) {
149 return 0, false
150 }
151 152 if r.codes[gerr.Code] {
153 return r.backoff.Pause(), true
154 }
155 156 return 0, false
157 }
158 159 // Backoff implements backoff logic for retries. The configuration for retries
160 // is described in https://google.aip.dev/client-libraries/4221. The current
161 // retry limit starts at Initial and increases by a factor of Multiplier every
162 // retry, but is capped at Max. The actual wait time between retries is a
163 // random value between 1ns and the current retry limit. The purpose of this
164 // random jitter is explained in
165 // https://www.awsarchitectureblog.com/2015/03/backoff.html.
166 //
167 // Note: MaxNumRetries / RPCDeadline is specifically not provided. These should
168 // be built on top of Backoff.
169 type Backoff struct {
170 // Initial is the initial value of the retry period, defaults to 1 second.
171 Initial time.Duration
172 173 // Max is the maximum value of the retry period, defaults to 30 seconds.
174 Max time.Duration
175 176 // Multiplier is the factor by which the retry period increases.
177 // It should be greater than 1 and defaults to 2.
178 Multiplier float64
179 180 // cur is the current retry period.
181 cur time.Duration
182 }
183 184 // Pause returns the next time.Duration that the caller should use to backoff.
185 func (bo *Backoff) Pause() time.Duration {
186 if bo.Initial == 0 {
187 bo.Initial = time.Second
188 }
189 if bo.cur == 0 {
190 bo.cur = bo.Initial
191 }
192 if bo.Max == 0 {
193 bo.Max = 30 * time.Second
194 }
195 if bo.Multiplier < 1 {
196 bo.Multiplier = 2
197 }
198 // Select a duration between 1ns and the current max. It might seem
199 // counterintuitive to have so much jitter, but
200 // https://www.awsarchitectureblog.com/2015/03/backoff.html argues that
201 // that is the best strategy.
202 d := time.Duration(1 + rand.Int63n(int64(bo.cur)))
203 bo.cur = time.Duration(float64(bo.cur) * bo.Multiplier)
204 if bo.cur > bo.Max {
205 bo.cur = bo.Max
206 }
207 return d
208 }
209 210 type grpcOpt []grpc.CallOption
211 212 func (o grpcOpt) Resolve(s *CallSettings) {
213 s.GRPC = o
214 }
215 216 type pathOpt struct {
217 p string
218 }
219 220 func (p pathOpt) Resolve(s *CallSettings) {
221 s.Path = p.p
222 }
223 224 type timeoutOpt struct {
225 t time.Duration
226 }
227 228 func (t timeoutOpt) Resolve(s *CallSettings) {
229 s.timeout = t.t
230 }
231 232 // WithPath applies a Path override to the HTTP-based APICall.
233 //
234 // This is for internal use only.
235 func WithPath(p string) CallOption {
236 return &pathOpt{p: p}
237 }
238 239 // WithGRPCOptions allows passing gRPC call options during client creation.
240 func WithGRPCOptions(opt ...grpc.CallOption) CallOption {
241 return grpcOpt(append([]grpc.CallOption(nil), opt...))
242 }
243 244 // WithTimeout is a convenience option for setting a context.WithTimeout on the
245 // singular context.Context used for **all** APICall attempts. Calculated from
246 // the start of the first APICall attempt.
247 // If the context.Context provided to Invoke already has a Deadline set, that
248 // will always be respected over the deadline calculated using this option.
249 func WithTimeout(t time.Duration) CallOption {
250 return &timeoutOpt{t: t}
251 }
252 253 // CallSettings allow fine-grained control over how calls are made.
254 type CallSettings struct {
255 // Retry returns a Retryer to be used to control retry logic of a method call.
256 // If Retry is nil or the returned Retryer is nil, the call will not be retried.
257 Retry func() Retryer
258 259 // CallOptions to be forwarded to GRPC.
260 GRPC []grpc.CallOption
261 262 // Path is an HTTP override for an APICall.
263 Path string
264 265 // Timeout defines the amount of time that Invoke has to complete.
266 // Unexported so it cannot be changed by the code in an APICall.
267 timeout time.Duration
268 }
269