1 // Copyright (c) HashiCorp, Inc.
2 // SPDX-License-Identifier: MPL-2.0
3 4 // Package retryablehttp provides a familiar HTTP client interface with
5 // automatic retries and exponential backoff. It is a thin wrapper over the
6 // standard net/http client library and exposes nearly the same public API.
7 // This makes retryablehttp very easy to drop into existing programs.
8 //
9 // retryablehttp performs automatic retries under certain conditions. Mainly, if
10 // an error is returned by the client (connection errors etc), or if a 500-range
11 // response is received, then a retry is invoked. Otherwise, the response is
12 // returned and left to the caller to interpret.
13 //
14 // Requests which take a request body should provide a non-nil function
15 // parameter. The best choice is to provide either a function satisfying
16 // ReaderFunc which provides multiple io.Readers in an efficient manner, a
17 // *bytes.Buffer (the underlying raw byte slice will be used) or a raw byte
18 // slice. As it is a reference type, and we will wrap it as needed by readers,
19 // we can efficiently re-use the request body without needing to copy it. If an
20 // io.Reader (such as a *bytes.Reader) is provided, the full body will be read
21 // prior to the first request, and will be efficiently re-used for any retries.
22 // ReadSeeker can be used, but some users have observed occasional data races
23 // between the net/http library and the Seek functionality of some
24 // implementations of ReadSeeker, so should be avoided if possible.
25 package retryablehttp
26 27 import (
28 "bytes"
29 "context"
30 "fmt"
31 "io"
32 "log"
33 "math"
34 "math/rand"
35 "net/http"
36 "net/url"
37 "os"
38 "regexp"
39 "strconv"
40 "strings"
41 "sync"
42 "time"
43 44 cleanhttp "github.com/hashicorp/go-cleanhttp"
45 )
46 47 var (
48 // Default retry configuration
49 defaultRetryWaitMin = 1 * time.Second
50 defaultRetryWaitMax = 30 * time.Second
51 defaultRetryMax = 4
52 53 // defaultLogger is the logger provided with defaultClient
54 defaultLogger = log.New(os.Stderr, "", log.LstdFlags)
55 56 // defaultClient is used for performing requests without explicitly making
57 // a new client. It is purposely private to avoid modifications.
58 defaultClient = NewClient()
59 60 // We need to consume response bodies to maintain http connections, but
61 // limit the size we consume to respReadLimit.
62 respReadLimit = int64(4096)
63 64 // timeNow sets the function that returns the current time.
65 // This defaults to time.Now. Changes to this should only be done in tests.
66 timeNow = time.Now
67 68 // A regular expression to match the error returned by net/http when the
69 // configured number of redirects is exhausted. This error isn't typed
70 // specifically so we resort to matching on the error string.
71 redirectsErrorRe = regexp.MustCompile(`stopped after \d+ redirects\z`)
72 73 // A regular expression to match the error returned by net/http when the
74 // scheme specified in the URL is invalid. This error isn't typed
75 // specifically so we resort to matching on the error string.
76 schemeErrorRe = regexp.MustCompile(`unsupported protocol scheme`)
77 78 // A regular expression to match the error returned by net/http when a
79 // request header or value is invalid. This error isn't typed
80 // specifically so we resort to matching on the error string.
81 invalidHeaderErrorRe = regexp.MustCompile(`invalid header`)
82 83 // A regular expression to match the error returned by net/http when the
84 // TLS certificate is not trusted. This error isn't typed
85 // specifically so we resort to matching on the error string.
86 notTrustedErrorRe = regexp.MustCompile(`certificate is not trusted`)
87 )
88 89 // ReaderFunc is the type of function that can be given natively to NewRequest
90 type ReaderFunc func() (io.Reader, error)
91 92 // ResponseHandlerFunc is a type of function that takes in a Response, and does something with it.
93 // The ResponseHandlerFunc is called when the HTTP client successfully receives a response and the
94 // CheckRetry function indicates that a retry of the base request is not necessary.
95 // If an error is returned from this function, the CheckRetry policy will be used to determine
96 // whether to retry the whole request (including this handler).
97 //
98 // Make sure to check status codes! Even if the request was completed it may have a non-2xx status code.
99 //
100 // The response body is not automatically closed. It must be closed either by the ResponseHandlerFunc or
101 // by the caller out-of-band. Failure to do so will result in a memory leak.
102 type ResponseHandlerFunc func(*http.Response) error
103 104 // LenReader is an interface implemented by many in-memory io.Reader's. Used
105 // for automatically sending the right Content-Length header when possible.
106 type LenReader interface {
107 Len() int
108 }
109 110 // Request wraps the metadata needed to create HTTP requests.
111 type Request struct {
112 // body is a seekable reader over the request body payload. This is
113 // used to rewind the request data in between retries.
114 body ReaderFunc
115 116 responseHandler ResponseHandlerFunc
117 118 // Embed an HTTP request directly. This makes a *Request act exactly
119 // like an *http.Request so that all meta methods are supported.
120 *http.Request
121 }
122 123 // WithContext returns wrapped Request with a shallow copy of underlying *http.Request
124 // with its context changed to ctx. The provided ctx must be non-nil.
125 func (r *Request) WithContext(ctx context.Context) *Request {
126 return &Request{
127 body: r.body,
128 responseHandler: r.responseHandler,
129 Request: r.Request.WithContext(ctx),
130 }
131 }
132 133 // SetResponseHandler allows setting the response handler.
134 func (r *Request) SetResponseHandler(fn ResponseHandlerFunc) {
135 r.responseHandler = fn
136 }
137 138 // BodyBytes allows accessing the request body. It is an analogue to
139 // http.Request's Body variable, but it returns a copy of the underlying data
140 // rather than consuming it.
141 //
142 // This function is not thread-safe; do not call it at the same time as another
143 // call, or at the same time this request is being used with Client.Do.
144 func (r *Request) BodyBytes() ([]byte, error) {
145 if r.body == nil {
146 return nil, nil
147 }
148 body, err := r.body()
149 if err != nil {
150 return nil, err
151 }
152 buf := new(bytes.Buffer)
153 _, err = buf.ReadFrom(body)
154 if err != nil {
155 return nil, err
156 }
157 return buf.Bytes(), nil
158 }
159 160 // SetBody allows setting the request body.
161 //
162 // It is useful if a new body needs to be set without constructing a new Request.
163 func (r *Request) SetBody(rawBody interface{}) error {
164 bodyReader, contentLength, err := getBodyReaderAndContentLength(rawBody)
165 if err != nil {
166 return err
167 }
168 r.body = bodyReader
169 r.ContentLength = contentLength
170 if bodyReader != nil {
171 r.GetBody = func() (io.ReadCloser, error) {
172 body, err := bodyReader()
173 if err != nil {
174 return nil, err
175 }
176 if rc, ok := body.(io.ReadCloser); ok {
177 return rc, nil
178 }
179 return io.NopCloser(body), nil
180 }
181 } else {
182 r.GetBody = func() (io.ReadCloser, error) { return http.NoBody, nil }
183 }
184 return nil
185 }
186 187 // WriteTo allows copying the request body into a writer.
188 //
189 // It writes data to w until there's no more data to write or
190 // when an error occurs. The return int64 value is the number of bytes
191 // written. Any error encountered during the write is also returned.
192 // The signature matches io.WriterTo interface.
193 func (r *Request) WriteTo(w io.Writer) (int64, error) {
194 body, err := r.body()
195 if err != nil {
196 return 0, err
197 }
198 if c, ok := body.(io.Closer); ok {
199 defer c.Close()
200 }
201 return io.Copy(w, body)
202 }
203 204 func getBodyReaderAndContentLength(rawBody interface{}) (ReaderFunc, int64, error) {
205 var bodyReader ReaderFunc
206 var contentLength int64
207 208 switch body := rawBody.(type) {
209 // If they gave us a function already, great! Use it.
210 case ReaderFunc:
211 bodyReader = body
212 tmp, err := body()
213 if err != nil {
214 return nil, 0, err
215 }
216 if lr, ok := tmp.(LenReader); ok {
217 contentLength = int64(lr.Len())
218 }
219 if c, ok := tmp.(io.Closer); ok {
220 c.Close()
221 }
222 223 case func() (io.Reader, error):
224 bodyReader = body
225 tmp, err := body()
226 if err != nil {
227 return nil, 0, err
228 }
229 if lr, ok := tmp.(LenReader); ok {
230 contentLength = int64(lr.Len())
231 }
232 if c, ok := tmp.(io.Closer); ok {
233 c.Close()
234 }
235 236 // If a regular byte slice, we can read it over and over via new
237 // readers
238 case []byte:
239 buf := body
240 bodyReader = func() (io.Reader, error) {
241 return bytes.NewReader(buf), nil
242 }
243 contentLength = int64(len(buf))
244 245 // If a bytes.Buffer we can read the underlying byte slice over and
246 // over
247 case *bytes.Buffer:
248 buf := body
249 bodyReader = func() (io.Reader, error) {
250 return bytes.NewReader(buf.Bytes()), nil
251 }
252 contentLength = int64(buf.Len())
253 254 // We prioritize *bytes.Reader here because we don't really want to
255 // deal with it seeking so want it to match here instead of the
256 // io.ReadSeeker case.
257 case *bytes.Reader:
258 snapshot := *body
259 bodyReader = func() (io.Reader, error) {
260 r := snapshot
261 return &r, nil
262 }
263 contentLength = int64(body.Len())
264 265 // Compat case
266 case io.ReadSeeker:
267 raw := body
268 bodyReader = func() (io.Reader, error) {
269 _, err := raw.Seek(0, 0)
270 return io.NopCloser(raw), err
271 }
272 if lr, ok := raw.(LenReader); ok {
273 contentLength = int64(lr.Len())
274 }
275 276 // Read all in so we can reset
277 case io.Reader:
278 buf, err := io.ReadAll(body)
279 if err != nil {
280 return nil, 0, err
281 }
282 if len(buf) == 0 {
283 bodyReader = func() (io.Reader, error) {
284 return http.NoBody, nil
285 }
286 contentLength = 0
287 } else {
288 bodyReader = func() (io.Reader, error) {
289 return bytes.NewReader(buf), nil
290 }
291 contentLength = int64(len(buf))
292 }
293 294 // No body provided, nothing to do
295 case nil:
296 297 // Unrecognized type
298 default:
299 return nil, 0, fmt.Errorf("cannot handle type %T", rawBody)
300 }
301 return bodyReader, contentLength, nil
302 }
303 304 // FromRequest wraps an http.Request in a retryablehttp.Request
305 func FromRequest(r *http.Request) (*Request, error) {
306 bodyReader, _, err := getBodyReaderAndContentLength(r.Body)
307 if err != nil {
308 return nil, err
309 }
310 // Could assert contentLength == r.ContentLength
311 return &Request{body: bodyReader, Request: r}, nil
312 }
313 314 // NewRequest creates a new wrapped request.
315 func NewRequest(method, url string, rawBody interface{}) (*Request, error) {
316 return NewRequestWithContext(context.Background(), method, url, rawBody)
317 }
318 319 // NewRequestWithContext creates a new wrapped request with the provided context.
320 //
321 // The context controls the entire lifetime of a request and its response:
322 // obtaining a connection, sending the request, and reading the response headers and body.
323 func NewRequestWithContext(ctx context.Context, method, url string, rawBody interface{}) (*Request, error) {
324 httpReq, err := http.NewRequestWithContext(ctx, method, url, nil)
325 if err != nil {
326 return nil, err
327 }
328 329 req := &Request{
330 Request: httpReq,
331 }
332 if err := req.SetBody(rawBody); err != nil {
333 return nil, err
334 }
335 336 return req, nil
337 }
338 339 // Logger interface allows to use other loggers than
340 // standard log.Logger.
341 type Logger interface {
342 Printf(string, ...interface{})
343 }
344 345 // LeveledLogger is an interface that can be implemented by any logger or a
346 // logger wrapper to provide leveled logging. The methods accept a message
347 // string and a variadic number of key-value pairs. For log.Printf style
348 // formatting where message string contains a format specifier, use Logger
349 // interface.
350 type LeveledLogger interface {
351 Error(msg string, keysAndValues ...interface{})
352 Info(msg string, keysAndValues ...interface{})
353 Debug(msg string, keysAndValues ...interface{})
354 Warn(msg string, keysAndValues ...interface{})
355 }
356 357 // hookLogger adapts an LeveledLogger to Logger for use by the existing hook functions
358 // without changing the API.
359 type hookLogger struct {
360 LeveledLogger
361 }
362 363 func (h hookLogger) Printf(s string, args ...interface{}) {
364 h.Info(fmt.Sprintf(s, args...))
365 }
366 367 // RequestLogHook allows a function to run before each retry. The HTTP
368 // request which will be made, and the retry number (0 for the initial
369 // request) are available to users. The internal logger is exposed to
370 // consumers.
371 type RequestLogHook func(Logger, *http.Request, int)
372 373 // ResponseLogHook is like RequestLogHook, but allows running a function
374 // on each HTTP response. This function will be invoked at the end of
375 // every HTTP request executed, regardless of whether a subsequent retry
376 // needs to be performed or not. If the response body is read or closed
377 // from this method, this will affect the response returned from Do().
378 type ResponseLogHook func(Logger, *http.Response)
379 380 // CheckRetry specifies a policy for handling retries. It is called
381 // following each request with the response and error values returned by
382 // the http.Client. If CheckRetry returns false, the Client stops retrying
383 // and returns the response to the caller. If CheckRetry returns an error,
384 // that error value is returned in lieu of the error from the request. The
385 // Client will close any response body when retrying, but if the retry is
386 // aborted it is up to the CheckRetry callback to properly close any
387 // response body before returning.
388 type CheckRetry func(ctx context.Context, resp *http.Response, err error) (bool, error)
389 390 // Backoff specifies a policy for how long to wait between retries.
391 // It is called after a failing request to determine the amount of time
392 // that should pass before trying again.
393 type Backoff func(min, max time.Duration, attemptNum int, resp *http.Response) time.Duration
394 395 // ErrorHandler is called if retries are expired, containing the last status
396 // from the http library. If not specified, default behavior for the library is
397 // to close the body and return an error indicating how many tries were
398 // attempted. If overriding this, be sure to close the body if needed.
399 type ErrorHandler func(resp *http.Response, err error, numTries int) (*http.Response, error)
400 401 // PrepareRetry is called before retry operation. It can be used for example to re-sign the request
402 type PrepareRetry func(req *http.Request) error
403 404 // Client is used to make HTTP requests. It adds additional functionality
405 // like automatic retries to tolerate minor outages.
406 type Client struct {
407 HTTPClient *http.Client // Internal HTTP client.
408 Logger interface{} // Customer logger instance. Can be either Logger or LeveledLogger
409 410 RetryWaitMin time.Duration // Minimum time to wait
411 RetryWaitMax time.Duration // Maximum time to wait
412 RetryMax int // Maximum number of retries
413 414 // RequestLogHook allows a user-supplied function to be called
415 // before each retry.
416 RequestLogHook RequestLogHook
417 418 // ResponseLogHook allows a user-supplied function to be called
419 // with the response from each HTTP request executed.
420 ResponseLogHook ResponseLogHook
421 422 // CheckRetry specifies the policy for handling retries, and is called
423 // after each request. The default policy is DefaultRetryPolicy.
424 CheckRetry CheckRetry
425 426 // Backoff specifies the policy for how long to wait between retries
427 Backoff Backoff
428 429 // ErrorHandler specifies the custom error handler to use, if any
430 ErrorHandler ErrorHandler
431 432 // PrepareRetry can prepare the request for retry operation, for example re-sign it
433 PrepareRetry PrepareRetry
434 435 loggerInit sync.Once
436 clientInit sync.Once
437 }
438 439 // NewClient creates a new Client with default settings.
440 func NewClient() *Client {
441 return &Client{
442 HTTPClient: cleanhttp.DefaultPooledClient(),
443 Logger: defaultLogger,
444 RetryWaitMin: defaultRetryWaitMin,
445 RetryWaitMax: defaultRetryWaitMax,
446 RetryMax: defaultRetryMax,
447 CheckRetry: DefaultRetryPolicy,
448 Backoff: DefaultBackoff,
449 }
450 }
451 452 func (c *Client) logger() interface{} {
453 c.loggerInit.Do(func() {
454 if c.Logger == nil {
455 return
456 }
457 458 switch c.Logger.(type) {
459 case Logger, LeveledLogger:
460 // ok
461 default:
462 // This should happen in dev when they are setting Logger and work on code, not in prod.
463 panic(fmt.Sprintf("invalid logger type passed, must be Logger or LeveledLogger, was %T", c.Logger))
464 }
465 })
466 467 return c.Logger
468 }
469 470 // DefaultRetryPolicy provides a default callback for Client.CheckRetry, which
471 // will retry on connection errors and server errors.
472 func DefaultRetryPolicy(ctx context.Context, resp *http.Response, err error) (bool, error) {
473 // do not retry on context.Canceled or context.DeadlineExceeded
474 if ctx.Err() != nil {
475 return false, ctx.Err()
476 }
477 478 // don't propagate other errors
479 shouldRetry, _ := baseRetryPolicy(resp, err)
480 return shouldRetry, nil
481 }
482 483 // ErrorPropagatedRetryPolicy is the same as DefaultRetryPolicy, except it
484 // propagates errors back instead of returning nil. This allows you to inspect
485 // why it decided to retry or not.
486 func ErrorPropagatedRetryPolicy(ctx context.Context, resp *http.Response, err error) (bool, error) {
487 // do not retry on context.Canceled or context.DeadlineExceeded
488 if ctx.Err() != nil {
489 return false, ctx.Err()
490 }
491 492 return baseRetryPolicy(resp, err)
493 }
494 495 func baseRetryPolicy(resp *http.Response, err error) (bool, error) {
496 if err != nil {
497 if v, ok := err.(*url.Error); ok {
498 // Don't retry if the error was due to too many redirects.
499 if redirectsErrorRe.MatchString(v.Error()) {
500 return false, v
501 }
502 503 // Don't retry if the error was due to an invalid protocol scheme.
504 if schemeErrorRe.MatchString(v.Error()) {
505 return false, v
506 }
507 508 // Don't retry if the error was due to an invalid header.
509 if invalidHeaderErrorRe.MatchString(v.Error()) {
510 return false, v
511 }
512 513 // Don't retry if the error was due to TLS cert verification failure.
514 if notTrustedErrorRe.MatchString(v.Error()) {
515 return false, v
516 }
517 if isCertError(v.Err) {
518 return false, v
519 }
520 }
521 522 // The error is likely recoverable so retry.
523 return true, nil
524 }
525 526 // 429 Too Many Requests is recoverable. Sometimes the server puts
527 // a Retry-After response header to indicate when the server is
528 // available to start processing request from client.
529 if resp.StatusCode == http.StatusTooManyRequests {
530 return true, nil
531 }
532 533 // Check the response code. We retry on 500-range responses to allow
534 // the server time to recover, as 500's are typically not permanent
535 // errors and may relate to outages on the server side. This will catch
536 // invalid response codes as well, like 0 and 999.
537 if resp.StatusCode == 0 || (resp.StatusCode >= 500 && resp.StatusCode != http.StatusNotImplemented) {
538 return true, fmt.Errorf("unexpected HTTP status %s", resp.Status)
539 }
540 541 return false, nil
542 }
543 544 // DefaultBackoff provides a default callback for Client.Backoff which
545 // will perform exponential backoff based on the attempt number and limited
546 // by the provided minimum and maximum durations.
547 //
548 // It also tries to parse Retry-After response header when a http.StatusTooManyRequests
549 // (HTTP Code 429) is found in the resp parameter. Hence it will return the number of
550 // seconds the server states it may be ready to process more requests from this client.
551 func DefaultBackoff(min, max time.Duration, attemptNum int, resp *http.Response) time.Duration {
552 if resp != nil {
553 if resp.StatusCode == http.StatusTooManyRequests || resp.StatusCode == http.StatusServiceUnavailable {
554 if sleep, ok := parseRetryAfterHeader(resp.Header["Retry-After"]); ok {
555 return sleep
556 }
557 }
558 }
559 560 mult := math.Pow(2, float64(attemptNum)) * float64(min)
561 sleep := time.Duration(mult)
562 if float64(sleep) != mult || sleep > max {
563 sleep = max
564 }
565 return sleep
566 }
567 568 // parseRetryAfterHeader parses the Retry-After header and returns the
569 // delay duration according to the spec: https://httpwg.org/specs/rfc7231.html#header.retry-after
570 // The bool returned will be true if the header was successfully parsed.
571 // Otherwise, the header was either not present, or was not parseable according to the spec.
572 //
573 // Retry-After headers come in two flavors: Seconds or HTTP-Date
574 //
575 // Examples:
576 // * Retry-After: Fri, 31 Dec 1999 23:59:59 GMT
577 // * Retry-After: 120
578 func parseRetryAfterHeader(headers []string) (time.Duration, bool) {
579 if len(headers) == 0 || headers[0] == "" {
580 return 0, false
581 }
582 header := headers[0]
583 // Retry-After: 120
584 if sleep, err := strconv.ParseInt(header, 10, 64); err == nil {
585 if sleep < 0 { // a negative sleep doesn't make sense
586 return 0, false
587 }
588 return time.Second * time.Duration(sleep), true
589 }
590 591 // Retry-After: Fri, 31 Dec 1999 23:59:59 GMT
592 retryTime, err := time.Parse(time.RFC1123, header)
593 if err != nil {
594 return 0, false
595 }
596 if until := retryTime.Sub(timeNow()); until > 0 {
597 return until, true
598 }
599 // date is in the past
600 return 0, true
601 }
602 603 // LinearJitterBackoff provides a callback for Client.Backoff which will
604 // perform linear backoff based on the attempt number and with jitter to
605 // prevent a thundering herd.
606 //
607 // min and max here are *not* absolute values. The number to be multiplied by
608 // the attempt number will be chosen at random from between them, thus they are
609 // bounding the jitter.
610 //
611 // For instance:
612 // * To get strictly linear backoff of one second increasing each retry, set
613 // both to one second (1s, 2s, 3s, 4s, ...)
614 // * To get a small amount of jitter centered around one second increasing each
615 // retry, set to around one second, such as a min of 800ms and max of 1200ms
616 // (892ms, 2102ms, 2945ms, 4312ms, ...)
617 // * To get extreme jitter, set to a very wide spread, such as a min of 100ms
618 // and a max of 20s (15382ms, 292ms, 51321ms, 35234ms, ...)
619 func LinearJitterBackoff(min, max time.Duration, attemptNum int, resp *http.Response) time.Duration {
620 // attemptNum always starts at zero but we want to start at 1 for multiplication
621 attemptNum++
622 623 if max <= min {
624 // Unclear what to do here, or they are the same, so return min *
625 // attemptNum
626 return min * time.Duration(attemptNum)
627 }
628 629 // Seed rand; doing this every time is fine
630 source := rand.New(rand.NewSource(int64(time.Now().Nanosecond())))
631 632 // Pick a random number that lies somewhere between the min and max and
633 // multiply by the attemptNum. attemptNum starts at zero so we always
634 // increment here. We first get a random percentage, then apply that to the
635 // difference between min and max, and add to min.
636 jitter := source.Float64() * float64(max-min)
637 jitterMin := int64(jitter) + int64(min)
638 return time.Duration(jitterMin * int64(attemptNum))
639 }
640 641 // RateLimitLinearJitterBackoff wraps the retryablehttp.LinearJitterBackoff.
642 // It first checks if the response status code is http.StatusTooManyRequests
643 // (HTTP Code 429) or http.StatusServiceUnavailable (HTTP Code 503). If it is
644 // and the response contains a Retry-After response header, it will wait the
645 // amount of time specified by the header. Otherwise, this calls
646 // LinearJitterBackoff.
647 func RateLimitLinearJitterBackoff(min, max time.Duration, attemptNum int, resp *http.Response) time.Duration {
648 if resp != nil {
649 if resp.StatusCode == http.StatusTooManyRequests || resp.StatusCode == http.StatusServiceUnavailable {
650 if sleep, ok := parseRetryAfterHeader(resp.Header["Retry-After"]); ok {
651 return sleep
652 }
653 }
654 }
655 return LinearJitterBackoff(min, max, attemptNum, resp)
656 }
657 658 // PassthroughErrorHandler is an ErrorHandler that directly passes through the
659 // values from the net/http library for the final request. The body is not
660 // closed.
661 func PassthroughErrorHandler(resp *http.Response, err error, _ int) (*http.Response, error) {
662 return resp, err
663 }
664 665 // Do wraps calling an HTTP method with retries.
666 func (c *Client) Do(req *Request) (*http.Response, error) {
667 c.clientInit.Do(func() {
668 if c.HTTPClient == nil {
669 c.HTTPClient = cleanhttp.DefaultPooledClient()
670 }
671 })
672 673 logger := c.logger()
674 675 if logger != nil {
676 switch v := logger.(type) {
677 case LeveledLogger:
678 v.Debug("performing request", "method", req.Method, "url", redactURL(req.URL))
679 case Logger:
680 v.Printf("[DEBUG] %s %s", req.Method, redactURL(req.URL))
681 }
682 }
683 684 var resp *http.Response
685 var attempt int
686 var shouldRetry bool
687 var doErr, respErr, checkErr, prepareErr error
688 689 for i := 0; ; i++ {
690 doErr, respErr, prepareErr = nil, nil, nil
691 attempt++
692 693 // Always rewind the request body when non-nil.
694 if req.body != nil {
695 body, err := req.body()
696 if err != nil {
697 c.HTTPClient.CloseIdleConnections()
698 return resp, err
699 }
700 if c, ok := body.(io.ReadCloser); ok {
701 req.Body = c
702 } else {
703 req.Body = io.NopCloser(body)
704 }
705 }
706 707 if c.RequestLogHook != nil {
708 switch v := logger.(type) {
709 case LeveledLogger:
710 c.RequestLogHook(hookLogger{v}, req.Request, i)
711 case Logger:
712 c.RequestLogHook(v, req.Request, i)
713 default:
714 c.RequestLogHook(nil, req.Request, i)
715 }
716 }
717 718 // Attempt the request
719 resp, doErr = c.HTTPClient.Do(req.Request)
720 721 // Check if we should continue with retries.
722 shouldRetry, checkErr = c.CheckRetry(req.Context(), resp, doErr)
723 if !shouldRetry && doErr == nil && req.responseHandler != nil {
724 respErr = req.responseHandler(resp)
725 shouldRetry, checkErr = c.CheckRetry(req.Context(), resp, respErr)
726 }
727 728 err := doErr
729 if respErr != nil {
730 err = respErr
731 }
732 if err != nil {
733 switch v := logger.(type) {
734 case LeveledLogger:
735 v.Error("request failed", "error", err, "method", req.Method, "url", redactURL(req.URL))
736 case Logger:
737 v.Printf("[ERR] %s %s request failed: %v", req.Method, redactURL(req.URL), err)
738 }
739 } else {
740 // Call this here to maintain the behavior of logging all requests,
741 // even if CheckRetry signals to stop.
742 if c.ResponseLogHook != nil {
743 // Call the response logger function if provided.
744 switch v := logger.(type) {
745 case LeveledLogger:
746 c.ResponseLogHook(hookLogger{v}, resp)
747 case Logger:
748 c.ResponseLogHook(v, resp)
749 default:
750 c.ResponseLogHook(nil, resp)
751 }
752 }
753 }
754 755 if !shouldRetry {
756 break
757 }
758 759 // We do this before drainBody because there's no need for the I/O if
760 // we're breaking out
761 remain := c.RetryMax - i
762 if remain <= 0 {
763 break
764 }
765 766 // We're going to retry, consume any response to reuse the connection.
767 if doErr == nil {
768 c.drainBody(resp.Body)
769 }
770 771 wait := c.Backoff(c.RetryWaitMin, c.RetryWaitMax, i, resp)
772 if logger != nil {
773 desc := fmt.Sprintf("%s %s", req.Method, redactURL(req.URL))
774 if resp != nil {
775 desc = fmt.Sprintf("%s (status: %d)", desc, resp.StatusCode)
776 }
777 switch v := logger.(type) {
778 case LeveledLogger:
779 v.Debug("retrying request", "request", desc, "timeout", wait, "remaining", remain)
780 case Logger:
781 v.Printf("[DEBUG] %s: retrying in %s (%d left)", desc, wait, remain)
782 }
783 }
784 timer := time.NewTimer(wait)
785 select {
786 case <-req.Context().Done():
787 timer.Stop()
788 c.HTTPClient.CloseIdleConnections()
789 return nil, req.Context().Err()
790 case <-timer.C:
791 }
792 793 // Make shallow copy of http Request so that we can modify its body
794 // without racing against the closeBody call in persistConn.writeLoop.
795 httpreq := *req.Request
796 req.Request = &httpreq
797 798 if c.PrepareRetry != nil {
799 if err := c.PrepareRetry(req.Request); err != nil {
800 prepareErr = err
801 break
802 }
803 }
804 }
805 806 // this is the closest we have to success criteria
807 if doErr == nil && respErr == nil && checkErr == nil && prepareErr == nil && !shouldRetry {
808 return resp, nil
809 }
810 811 defer c.HTTPClient.CloseIdleConnections()
812 813 var err error
814 if prepareErr != nil {
815 err = prepareErr
816 } else if checkErr != nil {
817 err = checkErr
818 } else if respErr != nil {
819 err = respErr
820 } else {
821 err = doErr
822 }
823 824 if c.ErrorHandler != nil {
825 return c.ErrorHandler(resp, err, attempt)
826 }
827 828 // By default, we close the response body and return an error without
829 // returning the response
830 if resp != nil {
831 c.drainBody(resp.Body)
832 }
833 834 // this means CheckRetry thought the request was a failure, but didn't
835 // communicate why
836 if err == nil {
837 return nil, fmt.Errorf("%s %s giving up after %d attempt(s)",
838 req.Method, redactURL(req.URL), attempt)
839 }
840 841 return nil, fmt.Errorf("%s %s giving up after %d attempt(s): %w",
842 req.Method, redactURL(req.URL), attempt, err)
843 }
844 845 // Try to read the response body so we can reuse this connection.
846 func (c *Client) drainBody(body io.ReadCloser) {
847 defer body.Close()
848 _, err := io.Copy(io.Discard, io.LimitReader(body, respReadLimit))
849 if err != nil {
850 if c.logger() != nil {
851 switch v := c.logger().(type) {
852 case LeveledLogger:
853 v.Error("error reading response body", "error", err)
854 case Logger:
855 v.Printf("[ERR] error reading response body: %v", err)
856 }
857 }
858 }
859 }
860 861 // Get is a shortcut for doing a GET request without making a new client.
862 func Get(url string) (*http.Response, error) {
863 return defaultClient.Get(url)
864 }
865 866 // Get is a convenience helper for doing simple GET requests.
867 func (c *Client) Get(url string) (*http.Response, error) {
868 req, err := NewRequest("GET", url, nil)
869 if err != nil {
870 return nil, err
871 }
872 return c.Do(req)
873 }
874 875 // Head is a shortcut for doing a HEAD request without making a new client.
876 func Head(url string) (*http.Response, error) {
877 return defaultClient.Head(url)
878 }
879 880 // Head is a convenience method for doing simple HEAD requests.
881 func (c *Client) Head(url string) (*http.Response, error) {
882 req, err := NewRequest("HEAD", url, nil)
883 if err != nil {
884 return nil, err
885 }
886 return c.Do(req)
887 }
888 889 // Post is a shortcut for doing a POST request without making a new client.
890 // The bodyType parameter sets the "Content-Type" header of the request.
891 func Post(url, bodyType string, body interface{}) (*http.Response, error) {
892 return defaultClient.Post(url, bodyType, body)
893 }
894 895 // Post is a convenience method for doing simple POST requests.
896 // The bodyType parameter sets the "Content-Type" header of the request.
897 func (c *Client) Post(url, bodyType string, body interface{}) (*http.Response, error) {
898 req, err := NewRequest("POST", url, body)
899 if err != nil {
900 return nil, err
901 }
902 req.Header.Set("Content-Type", bodyType)
903 return c.Do(req)
904 }
905 906 // PostForm is a shortcut to perform a POST with form data without creating
907 // a new client.
908 func PostForm(url string, data url.Values) (*http.Response, error) {
909 return defaultClient.PostForm(url, data)
910 }
911 912 // PostForm is a convenience method for doing simple POST operations using
913 // pre-filled url.Values form data.
914 func (c *Client) PostForm(url string, data url.Values) (*http.Response, error) {
915 return c.Post(url, "application/x-www-form-urlencoded", strings.NewReader(data.Encode()))
916 }
917 918 // StandardClient returns a stdlib *http.Client with a custom Transport, which
919 // shims in a *retryablehttp.Client for added retries.
920 func (c *Client) StandardClient() *http.Client {
921 return &http.Client{
922 Transport: &RoundTripper{Client: c},
923 }
924 }
925 926 // Taken from url.URL#Redacted() which was introduced in go 1.15.
927 // We can switch to using it directly if we'll bump the minimum required go version.
928 func redactURL(u *url.URL) string {
929 if u == nil {
930 return ""
931 }
932 933 ru := *u
934 if _, has := ru.User.Password(); has {
935 ru.User = url.UserPassword(ru.User.Username(), "xxxxx")
936 }
937 return ru.String()
938 }
939