1 package autorest
2 3 // Copyright 2017 Microsoft Corporation
4 //
5 // Licensed under the Apache License, Version 2.0 (the "License");
6 // you may not use this file except in compliance with the License.
7 // You may obtain a copy of the License at
8 //
9 // http://www.apache.org/licenses/LICENSE-2.0
10 //
11 // Unless required by applicable law or agreed to in writing, software
12 // distributed under the License is distributed on an "AS IS" BASIS,
13 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 // See the License for the specific language governing permissions and
15 // limitations under the License.
16 17 import (
18 "context"
19 "crypto/tls"
20 "fmt"
21 "log"
22 "math"
23 "net"
24 "net/http"
25 "net/http/cookiejar"
26 "strconv"
27 "sync"
28 "time"
29 30 "github.com/Azure/go-autorest/logger"
31 "github.com/Azure/go-autorest/tracing"
32 )
33 34 // there is one sender per TLS renegotiation type, i.e. count of tls.RenegotiationSupport enums
35 const defaultSendersCount = 3
36 37 type defaultSender struct {
38 sender Sender
39 init *sync.Once
40 }
41 42 // each type of sender will be created on demand in sender()
43 var defaultSenders [defaultSendersCount]defaultSender
44 45 func init() {
46 for i := 0; i < defaultSendersCount; i++ {
47 defaultSenders[i].init = &sync.Once{}
48 }
49 }
50 51 // used as a key type in context.WithValue()
52 type ctxSendDecorators struct{}
53 54 // WithSendDecorators adds the specified SendDecorators to the provided context.
55 // If no SendDecorators are provided the context is unchanged.
56 func WithSendDecorators(ctx context.Context, sendDecorator []SendDecorator) context.Context {
57 if len(sendDecorator) == 0 {
58 return ctx
59 }
60 return context.WithValue(ctx, ctxSendDecorators{}, sendDecorator)
61 }
62 63 // GetSendDecorators returns the SendDecorators in the provided context or the provided default SendDecorators.
64 func GetSendDecorators(ctx context.Context, defaultSendDecorators ...SendDecorator) []SendDecorator {
65 inCtx := ctx.Value(ctxSendDecorators{})
66 if sd, ok := inCtx.([]SendDecorator); ok {
67 return sd
68 }
69 return defaultSendDecorators
70 }
71 72 // Sender is the interface that wraps the Do method to send HTTP requests.
73 //
74 // The standard http.Client conforms to this interface.
75 type Sender interface {
76 Do(*http.Request) (*http.Response, error)
77 }
78 79 // SenderFunc is a method that implements the Sender interface.
80 type SenderFunc func(*http.Request) (*http.Response, error)
81 82 // Do implements the Sender interface on SenderFunc.
83 func (sf SenderFunc) Do(r *http.Request) (*http.Response, error) {
84 return sf(r)
85 }
86 87 // SendDecorator takes and possibly decorates, by wrapping, a Sender. Decorators may affect the
88 // http.Request and pass it along or, first, pass the http.Request along then react to the
89 // http.Response result.
90 type SendDecorator func(Sender) Sender
91 92 // CreateSender creates, decorates, and returns, as a Sender, the default http.Client.
93 func CreateSender(decorators ...SendDecorator) Sender {
94 return DecorateSender(sender(tls.RenegotiateNever), decorators...)
95 }
96 97 // DecorateSender accepts a Sender and a, possibly empty, set of SendDecorators, which is applies to
98 // the Sender. Decorators are applied in the order received, but their affect upon the request
99 // depends on whether they are a pre-decorator (change the http.Request and then pass it along) or a
100 // post-decorator (pass the http.Request along and react to the results in http.Response).
101 func DecorateSender(s Sender, decorators ...SendDecorator) Sender {
102 for _, decorate := range decorators {
103 s = decorate(s)
104 }
105 return s
106 }
107 108 // Send sends, by means of the default http.Client, the passed http.Request, returning the
109 // http.Response and possible error. It also accepts a, possibly empty, set of SendDecorators which
110 // it will apply the http.Client before invoking the Do method.
111 //
112 // Send is a convenience method and not recommended for production. Advanced users should use
113 // SendWithSender, passing and sharing their own Sender (e.g., instance of http.Client).
114 //
115 // Send will not poll or retry requests.
116 func Send(r *http.Request, decorators ...SendDecorator) (*http.Response, error) {
117 return SendWithSender(sender(tls.RenegotiateNever), r, decorators...)
118 }
119 120 // SendWithSender sends the passed http.Request, through the provided Sender, returning the
121 // http.Response and possible error. It also accepts a, possibly empty, set of SendDecorators which
122 // it will apply the http.Client before invoking the Do method.
123 //
124 // SendWithSender will not poll or retry requests.
125 func SendWithSender(s Sender, r *http.Request, decorators ...SendDecorator) (*http.Response, error) {
126 return DecorateSender(s, decorators...).Do(r)
127 }
128 129 func sender(renengotiation tls.RenegotiationSupport) Sender {
130 // note that we can't init defaultSenders in init() since it will
131 // execute before calling code has had a chance to enable tracing
132 defaultSenders[renengotiation].init.Do(func() {
133 // copied from http.DefaultTransport with a TLS minimum version.
134 transport := &http.Transport{
135 Proxy: http.ProxyFromEnvironment,
136 DialContext: (&net.Dialer{
137 Timeout: 30 * time.Second,
138 KeepAlive: 30 * time.Second,
139 }).DialContext,
140 ForceAttemptHTTP2: true,
141 MaxIdleConns: 100,
142 IdleConnTimeout: 90 * time.Second,
143 TLSHandshakeTimeout: 10 * time.Second,
144 ExpectContinueTimeout: 1 * time.Second,
145 TLSClientConfig: &tls.Config{
146 MinVersion: tls.VersionTLS12,
147 Renegotiation: renengotiation,
148 },
149 }
150 var roundTripper http.RoundTripper = transport
151 if tracing.IsEnabled() {
152 roundTripper = tracing.NewTransport(transport)
153 }
154 j, _ := cookiejar.New(nil)
155 defaultSenders[renengotiation].sender = &http.Client{Jar: j, Transport: roundTripper}
156 })
157 return defaultSenders[renengotiation].sender
158 }
159 160 // AfterDelay returns a SendDecorator that delays for the passed time.Duration before
161 // invoking the Sender. The delay may be terminated by closing the optional channel on the
162 // http.Request. If canceled, no further Senders are invoked.
163 func AfterDelay(d time.Duration) SendDecorator {
164 return func(s Sender) Sender {
165 return SenderFunc(func(r *http.Request) (*http.Response, error) {
166 if !DelayForBackoff(d, 0, r.Context().Done()) {
167 return nil, fmt.Errorf("autorest: AfterDelay canceled before full delay")
168 }
169 return s.Do(r)
170 })
171 }
172 }
173 174 // AsIs returns a SendDecorator that invokes the passed Sender without modifying the http.Request.
175 func AsIs() SendDecorator {
176 return func(s Sender) Sender {
177 return SenderFunc(func(r *http.Request) (*http.Response, error) {
178 return s.Do(r)
179 })
180 }
181 }
182 183 // DoCloseIfError returns a SendDecorator that first invokes the passed Sender after which
184 // it closes the response if the passed Sender returns an error and the response body exists.
185 func DoCloseIfError() SendDecorator {
186 return func(s Sender) Sender {
187 return SenderFunc(func(r *http.Request) (*http.Response, error) {
188 resp, err := s.Do(r)
189 if err != nil {
190 Respond(resp, ByDiscardingBody(), ByClosing())
191 }
192 return resp, err
193 })
194 }
195 }
196 197 // DoErrorIfStatusCode returns a SendDecorator that emits an error if the response StatusCode is
198 // among the set passed. Since these are artificial errors, the response body may still require
199 // closing.
200 func DoErrorIfStatusCode(codes ...int) SendDecorator {
201 return func(s Sender) Sender {
202 return SenderFunc(func(r *http.Request) (*http.Response, error) {
203 resp, err := s.Do(r)
204 if err == nil && ResponseHasStatusCode(resp, codes...) {
205 err = NewErrorWithResponse("autorest", "DoErrorIfStatusCode", resp, "%v %v failed with %s",
206 resp.Request.Method,
207 resp.Request.URL,
208 resp.Status)
209 }
210 return resp, err
211 })
212 }
213 }
214 215 // DoErrorUnlessStatusCode returns a SendDecorator that emits an error unless the response
216 // StatusCode is among the set passed. Since these are artificial errors, the response body
217 // may still require closing.
218 func DoErrorUnlessStatusCode(codes ...int) SendDecorator {
219 return func(s Sender) Sender {
220 return SenderFunc(func(r *http.Request) (*http.Response, error) {
221 resp, err := s.Do(r)
222 if err == nil && !ResponseHasStatusCode(resp, codes...) {
223 err = NewErrorWithResponse("autorest", "DoErrorUnlessStatusCode", resp, "%v %v failed with %s",
224 resp.Request.Method,
225 resp.Request.URL,
226 resp.Status)
227 }
228 return resp, err
229 })
230 }
231 }
232 233 // DoPollForStatusCodes returns a SendDecorator that polls if the http.Response contains one of the
234 // passed status codes. It expects the http.Response to contain a Location header providing the
235 // URL at which to poll (using GET) and will poll until the time passed is equal to or greater than
236 // the supplied duration. It will delay between requests for the duration specified in the
237 // RetryAfter header or, if the header is absent, the passed delay. Polling may be canceled by
238 // closing the optional channel on the http.Request.
239 func DoPollForStatusCodes(duration time.Duration, delay time.Duration, codes ...int) SendDecorator {
240 return func(s Sender) Sender {
241 return SenderFunc(func(r *http.Request) (resp *http.Response, err error) {
242 resp, err = s.Do(r)
243 244 if err == nil && ResponseHasStatusCode(resp, codes...) {
245 r, err = NewPollingRequestWithContext(r.Context(), resp)
246 247 for err == nil && ResponseHasStatusCode(resp, codes...) {
248 Respond(resp,
249 ByDiscardingBody(),
250 ByClosing())
251 resp, err = SendWithSender(s, r,
252 AfterDelay(GetRetryAfter(resp, delay)))
253 }
254 }
255 256 return resp, err
257 })
258 }
259 }
260 261 // DoRetryForAttempts returns a SendDecorator that retries a failed request for up to the specified
262 // number of attempts, exponentially backing off between requests using the supplied backoff
263 // time.Duration (which may be zero). Retrying may be canceled by closing the optional channel on
264 // the http.Request.
265 func DoRetryForAttempts(attempts int, backoff time.Duration) SendDecorator {
266 return func(s Sender) Sender {
267 return SenderFunc(func(r *http.Request) (resp *http.Response, err error) {
268 rr := NewRetriableRequest(r)
269 for attempt := 0; attempt < attempts; attempt++ {
270 err = rr.Prepare()
271 if err != nil {
272 return resp, err
273 }
274 DrainResponseBody(resp)
275 resp, err = s.Do(rr.Request())
276 if err == nil {
277 return resp, err
278 }
279 logger.Instance.Writef(logger.LogError, "DoRetryForAttempts: received error for attempt %d: %v\n", attempt+1, err)
280 if !DelayForBackoff(backoff, attempt, r.Context().Done()) {
281 return nil, r.Context().Err()
282 }
283 }
284 return resp, err
285 })
286 }
287 }
288 289 // Count429AsRetry indicates that a 429 response should be included as a retry attempt.
290 var Count429AsRetry = true
291 292 // Max429Delay is the maximum duration to wait between retries on a 429 if no Retry-After header was received.
293 var Max429Delay time.Duration
294 295 // DoRetryForStatusCodes returns a SendDecorator that retries for specified statusCodes for up to the specified
296 // number of attempts, exponentially backing off between requests using the supplied backoff
297 // time.Duration (which may be zero). Retrying may be canceled by cancelling the context on the http.Request.
298 // NOTE: Code http.StatusTooManyRequests (429) will *not* be counted against the number of attempts.
299 func DoRetryForStatusCodes(attempts int, backoff time.Duration, codes ...int) SendDecorator {
300 return func(s Sender) Sender {
301 return SenderFunc(func(r *http.Request) (*http.Response, error) {
302 return doRetryForStatusCodesImpl(s, r, Count429AsRetry, attempts, backoff, 0, codes...)
303 })
304 }
305 }
306 307 // DoRetryForStatusCodesWithCap returns a SendDecorator that retries for specified statusCodes for up to the
308 // specified number of attempts, exponentially backing off between requests using the supplied backoff
309 // time.Duration (which may be zero). To cap the maximum possible delay between iterations specify a value greater
310 // than zero for cap. Retrying may be canceled by cancelling the context on the http.Request.
311 func DoRetryForStatusCodesWithCap(attempts int, backoff, cap time.Duration, codes ...int) SendDecorator {
312 return func(s Sender) Sender {
313 return SenderFunc(func(r *http.Request) (*http.Response, error) {
314 return doRetryForStatusCodesImpl(s, r, Count429AsRetry, attempts, backoff, cap, codes...)
315 })
316 }
317 }
318 319 func doRetryForStatusCodesImpl(s Sender, r *http.Request, count429 bool, attempts int, backoff, cap time.Duration, codes ...int) (resp *http.Response, err error) {
320 rr := NewRetriableRequest(r)
321 // Increment to add the first call (attempts denotes number of retries)
322 for attempt, delayCount := 0, 0; attempt < attempts+1; {
323 err = rr.Prepare()
324 if err != nil {
325 return
326 }
327 DrainResponseBody(resp)
328 resp, err = s.Do(rr.Request())
329 // we want to retry if err is not nil (e.g. transient network failure). note that for failed authentication
330 // resp and err will both have a value, so in this case we don't want to retry as it will never succeed.
331 if err == nil && !ResponseHasStatusCode(resp, codes...) || IsTokenRefreshError(err) {
332 return resp, err
333 }
334 if err != nil {
335 logger.Instance.Writef(logger.LogError, "DoRetryForStatusCodes: received error for attempt %d: %v\n", attempt+1, err)
336 }
337 delayed := DelayWithRetryAfter(resp, r.Context().Done())
338 // if this was a 429 set the delay cap as specified.
339 // applicable only in the absence of a retry-after header.
340 if resp != nil && resp.StatusCode == http.StatusTooManyRequests {
341 cap = Max429Delay
342 }
343 if !delayed && !DelayForBackoffWithCap(backoff, cap, delayCount, r.Context().Done()) {
344 return resp, r.Context().Err()
345 }
346 // when count429 == false don't count a 429 against the number
347 // of attempts so that we continue to retry until it succeeds
348 if count429 || (resp == nil || resp.StatusCode != http.StatusTooManyRequests) {
349 attempt++
350 }
351 // delay count is tracked separately from attempts to
352 // ensure that 429 participates in exponential back-off
353 delayCount++
354 }
355 return resp, err
356 }
357 358 // DelayWithRetryAfter invokes time.After for the duration specified in the "Retry-After" header.
359 // The value of Retry-After can be either the number of seconds or a date in RFC1123 format.
360 // The function returns true after successfully waiting for the specified duration. If there is
361 // no Retry-After header or the wait is cancelled the return value is false.
362 func DelayWithRetryAfter(resp *http.Response, cancel <-chan struct{}) bool {
363 if resp == nil {
364 return false
365 }
366 var dur time.Duration
367 ra := resp.Header.Get("Retry-After")
368 if retryAfter, _ := strconv.Atoi(ra); retryAfter > 0 {
369 dur = time.Duration(retryAfter) * time.Second
370 } else if t, err := time.Parse(time.RFC1123, ra); err == nil {
371 dur = t.Sub(time.Now())
372 }
373 if dur > 0 {
374 select {
375 case <-time.After(dur):
376 return true
377 case <-cancel:
378 return false
379 }
380 }
381 return false
382 }
383 384 // DoRetryForDuration returns a SendDecorator that retries the request until the total time is equal
385 // to or greater than the specified duration, exponentially backing off between requests using the
386 // supplied backoff time.Duration (which may be zero). Retrying may be canceled by closing the
387 // optional channel on the http.Request.
388 func DoRetryForDuration(d time.Duration, backoff time.Duration) SendDecorator {
389 return func(s Sender) Sender {
390 return SenderFunc(func(r *http.Request) (resp *http.Response, err error) {
391 rr := NewRetriableRequest(r)
392 end := time.Now().Add(d)
393 for attempt := 0; time.Now().Before(end); attempt++ {
394 err = rr.Prepare()
395 if err != nil {
396 return resp, err
397 }
398 DrainResponseBody(resp)
399 resp, err = s.Do(rr.Request())
400 if err == nil {
401 return resp, err
402 }
403 logger.Instance.Writef(logger.LogError, "DoRetryForDuration: received error for attempt %d: %v\n", attempt+1, err)
404 if !DelayForBackoff(backoff, attempt, r.Context().Done()) {
405 return nil, r.Context().Err()
406 }
407 }
408 return resp, err
409 })
410 }
411 }
412 413 // WithLogging returns a SendDecorator that implements simple before and after logging of the
414 // request.
415 func WithLogging(logger *log.Logger) SendDecorator {
416 return func(s Sender) Sender {
417 return SenderFunc(func(r *http.Request) (*http.Response, error) {
418 logger.Printf("Sending %s %s", r.Method, r.URL)
419 resp, err := s.Do(r)
420 if err != nil {
421 logger.Printf("%s %s received error '%v'", r.Method, r.URL, err)
422 } else {
423 logger.Printf("%s %s received %s", r.Method, r.URL, resp.Status)
424 }
425 return resp, err
426 })
427 }
428 }
429 430 // DelayForBackoff invokes time.After for the supplied backoff duration raised to the power of
431 // passed attempt (i.e., an exponential backoff delay). Backoff duration is in seconds and can set
432 // to zero for no delay. The delay may be canceled by closing the passed channel. If terminated early,
433 // returns false.
434 // Note: Passing attempt 1 will result in doubling "backoff" duration. Treat this as a zero-based attempt
435 // count.
436 func DelayForBackoff(backoff time.Duration, attempt int, cancel <-chan struct{}) bool {
437 return DelayForBackoffWithCap(backoff, 0, attempt, cancel)
438 }
439 440 // DelayForBackoffWithCap invokes time.After for the supplied backoff duration raised to the power of
441 // passed attempt (i.e., an exponential backoff delay). Backoff duration is in seconds and can set
442 // to zero for no delay. To cap the maximum possible delay specify a value greater than zero for cap.
443 // The delay may be canceled by closing the passed channel. If terminated early, returns false.
444 // Note: Passing attempt 1 will result in doubling "backoff" duration. Treat this as a zero-based attempt
445 // count.
446 func DelayForBackoffWithCap(backoff, cap time.Duration, attempt int, cancel <-chan struct{}) bool {
447 d := time.Duration(backoff.Seconds()*math.Pow(2, float64(attempt))) * time.Second
448 if cap > 0 && d > cap {
449 d = cap
450 }
451 logger.Instance.Writef(logger.LogInfo, "DelayForBackoffWithCap: sleeping for %s\n", d)
452 select {
453 case <-time.After(d):
454 return true
455 case <-cancel:
456 return false
457 }
458 }
459