retries.go raw

   1  package linodego
   2  
   3  import (
   4  	"errors"
   5  	"log"
   6  	"net/http"
   7  	"strconv"
   8  	"time"
   9  
  10  	"github.com/go-resty/resty/v2"
  11  	"golang.org/x/net/http2"
  12  )
  13  
  14  const (
  15  	retryAfterHeaderName      = "Retry-After"
  16  	maintenanceModeHeaderName = "X-Maintenance-Mode"
  17  
  18  	defaultRetryCount = 1000
  19  )
  20  
  21  // RetryConditional func(r *resty.Response) (shouldRetry bool)
  22  type RetryConditional resty.RetryConditionFunc
  23  
  24  // RetryAfter func(c *resty.Client, r *resty.Response) (time.Duration, error)
  25  type RetryAfter resty.RetryAfterFunc
  26  
  27  // Configures resty to
  28  // lock until enough time has passed to retry the request as determined by the Retry-After response header.
  29  // If the Retry-After header is not set, we fall back to value of SetPollDelay.
  30  func configureRetries(c *Client) {
  31  	c.resty.
  32  		SetRetryCount(defaultRetryCount).
  33  		AddRetryCondition(checkRetryConditionals(c)).
  34  		SetRetryAfter(respectRetryAfter)
  35  }
  36  
  37  func checkRetryConditionals(c *Client) func(*resty.Response, error) bool {
  38  	return func(r *resty.Response, err error) bool {
  39  		for _, retryConditional := range c.retryConditionals {
  40  			retry := retryConditional(r, err)
  41  			if retry {
  42  				log.Printf("[INFO] Received error %s - Retrying", r.Error())
  43  				return true
  44  			}
  45  		}
  46  
  47  		return false
  48  	}
  49  }
  50  
  51  // SetLinodeBusyRetry configures resty to retry specifically on "Linode busy." errors
  52  // The retry wait time is configured in SetPollDelay
  53  func linodeBusyRetryCondition(r *resty.Response, _ error) bool {
  54  	apiError, ok := r.Error().(*APIError)
  55  	linodeBusy := ok && apiError.Error() == "Linode busy."
  56  	retry := r.StatusCode() == http.StatusBadRequest && linodeBusy
  57  
  58  	return retry
  59  }
  60  
  61  func tooManyRequestsRetryCondition(r *resty.Response, _ error) bool {
  62  	return r.StatusCode() == http.StatusTooManyRequests
  63  }
  64  
  65  func serviceUnavailableRetryCondition(r *resty.Response, _ error) bool {
  66  	serviceUnavailable := r.StatusCode() == http.StatusServiceUnavailable
  67  
  68  	// During maintenance events, the API will return a 503 and add
  69  	// an `X-MAINTENANCE-MODE` header. Don't retry during maintenance
  70  	// events, only for legitimate 503s.
  71  	if serviceUnavailable && r.Header().Get(maintenanceModeHeaderName) != "" {
  72  		log.Printf("[INFO] Linode API is under maintenance, request will not be retried - please see status.linode.com for more information")
  73  		return false
  74  	}
  75  
  76  	return serviceUnavailable
  77  }
  78  
  79  func requestTimeoutRetryCondition(r *resty.Response, _ error) bool {
  80  	return r.StatusCode() == http.StatusRequestTimeout
  81  }
  82  
  83  func requestGOAWAYRetryCondition(_ *resty.Response, e error) bool {
  84  	return errors.As(e, &http2.GoAwayError{})
  85  }
  86  
  87  func requestNGINXRetryCondition(r *resty.Response, _ error) bool {
  88  	return r.StatusCode() == http.StatusBadRequest &&
  89  		r.Header().Get("Server") == "nginx" &&
  90  		r.Header().Get("Content-Type") == "text/html"
  91  }
  92  
  93  func respectRetryAfter(client *resty.Client, resp *resty.Response) (time.Duration, error) {
  94  	retryAfterStr := resp.Header().Get(retryAfterHeaderName)
  95  	if retryAfterStr == "" {
  96  		return 0, nil
  97  	}
  98  
  99  	retryAfter, err := strconv.Atoi(retryAfterStr)
 100  	if err != nil {
 101  		return 0, err
 102  	}
 103  
 104  	duration := time.Duration(retryAfter) * time.Second
 105  	log.Printf("[INFO] Respecting Retry-After Header of %d (%s) (max %s)", retryAfter, duration, client.RetryMaxWaitTime)
 106  
 107  	return duration, nil
 108  }
 109