1 // Copyright (c) 2016, 2018, 2025, Oracle and/or its affiliates. All rights reserved.
2 // This software is dual-licensed to you under the Universal Permissive License (UPL) 1.0 as shown at https://oss.oracle.com/licenses/upl or Apache License 2.0 as shown at http://www.apache.org/licenses/LICENSE-2.0. You may choose either license.
3 4 package common
5 6 import (
7 "context"
8 "errors"
9 "fmt"
10 "io"
11 "math"
12 "math/rand"
13 "runtime"
14 "strings"
15 "time"
16 )
17 18 const (
19 // UnlimitedNumAttemptsValue is the value for indicating unlimited attempts for reaching success
20 UnlimitedNumAttemptsValue = uint(0)
21 22 // number of characters contained in the generated retry token
23 generatedRetryTokenLength = 32
24 )
25 26 // OCIRetryableRequest represents a request that can be reissued according to the specified policy.
27 type OCIRetryableRequest interface {
28 // Any retryable request must implement the OCIRequest interface
29 OCIRequest
30 31 // Each operation should implement this method, if has binary body, return OCIReadSeekCloser and true, otherwise return nil, false
32 BinaryRequestBody() (*OCIReadSeekCloser, bool)
33 34 // Each operation specifies default retry behavior. By passing no arguments to this method, the default retry
35 // behavior, as determined on a per-operation-basis, will be honored. Variadic retry policy option arguments
36 // passed to this method will override the default behavior.
37 RetryPolicy() *RetryPolicy
38 }
39 40 // OCIOperationResponse represents the output of an OCIOperation, with additional context of error message
41 // and operation attempt number.
42 type OCIOperationResponse struct {
43 // Response from OCI Operation
44 Response OCIResponse
45 46 // Error from OCI Operation
47 Error error
48 49 // Operation Attempt Number (one-based)
50 AttemptNumber uint
51 52 // End of eventually consistent effects, or nil if no such effects
53 EndOfWindowTime *time.Time
54 55 // Backoff scaling factor (only used for dealing with eventual consistency)
56 BackoffScalingFactor float64
57 58 // Time of the initial attempt
59 InitialAttemptTime time.Time
60 }
61 62 const (
63 defaultMaximumNumberAttempts = uint(8)
64 defaultExponentialBackoffBase = 2.0
65 defaultMinSleepBetween = 0.0
66 defaultMaxSleepBetween = 30.0
67 68 ecMaximumNumberAttempts = uint(9)
69 ecExponentialBackoffBase = 3.52
70 ecMinSleepBetween = 0.0
71 ecMaxSleepBetween = 45.0
72 )
73 74 var (
75 defaultRetryStatusCodeMap = map[StatErrCode]bool{
76 {409, "IncorrectState"}: true,
77 {429, "TooManyRequests"}: true,
78 79 {501, "MethodNotImplemented"}: false,
80 }
81 )
82 83 // IsErrorRetryableByDefault returns true if the error is retryable by OCI default retry policy
84 func IsErrorRetryableByDefault(err error) bool {
85 if err == nil {
86 return false
87 }
88 89 if IsNetworkError(err) {
90 return true
91 }
92 93 if err == io.EOF {
94 return true
95 }
96 97 if err, ok := IsServiceError(err); ok {
98 if shouldRetry, ok := defaultRetryStatusCodeMap[StatErrCode{err.GetHTTPStatusCode(), err.GetCode()}]; ok {
99 return shouldRetry
100 }
101 102 return 500 <= err.GetHTTPStatusCode() && err.GetHTTPStatusCode() < 505
103 }
104 105 return false
106 }
107 108 // NewOCIOperationResponse assembles an OCI Operation Response object.
109 // Note that InitialAttemptTime is not set, nor is EndOfWindowTime, and BackoffScalingFactor is set to 1.0.
110 // EndOfWindowTime and BackoffScalingFactor are only important for eventual consistency.
111 // InitialAttemptTime can be useful for time-based (as opposed to count-based) retry policies.
112 func NewOCIOperationResponse(response OCIResponse, err error, attempt uint) OCIOperationResponse {
113 return OCIOperationResponse{
114 Response: response,
115 Error: err,
116 AttemptNumber: attempt,
117 BackoffScalingFactor: 1.0,
118 }
119 }
120 121 // NewOCIOperationResponseExtended assembles an OCI Operation Response object, with the value for the EndOfWindowTime, BackoffScalingFactor, and InitialAttemptTime set.
122 // EndOfWindowTime and BackoffScalingFactor are only important for eventual consistency.
123 // InitialAttemptTime can be useful for time-based (as opposed to count-based) retry policies.
124 func NewOCIOperationResponseExtended(response OCIResponse, err error, attempt uint, endOfWindowTime *time.Time, backoffScalingFactor float64,
125 initialAttemptTime time.Time) OCIOperationResponse {
126 return OCIOperationResponse{
127 Response: response,
128 Error: err,
129 AttemptNumber: attempt,
130 EndOfWindowTime: endOfWindowTime,
131 BackoffScalingFactor: backoffScalingFactor,
132 InitialAttemptTime: initialAttemptTime,
133 }
134 }
135 136 //
137 // RetryPolicy
138 //
139 140 // RetryPolicy is the class that holds all relevant information for retrying operations.
141 type RetryPolicy struct {
142 // MaximumNumberAttempts is the maximum number of times to retry a request. Zero indicates an unlimited
143 // number of attempts.
144 MaximumNumberAttempts uint
145 146 // ShouldRetryOperation inspects the http response, error, and operation attempt number, and
147 // - returns true if we should retry the operation
148 // - returns false otherwise
149 ShouldRetryOperation func(OCIOperationResponse) bool
150 151 // GetNextDuration computes the duration to pause between operation retries.
152 NextDuration func(OCIOperationResponse) time.Duration
153 154 // minimum sleep between attempts in seconds
155 MinSleepBetween float64
156 157 // maximum sleep between attempts in seconds
158 MaxSleepBetween float64
159 160 // the base for the exponential backoff
161 ExponentialBackoffBase float64
162 163 // DeterminePolicyToUse may modify the policy to handle eventual consistency; the return values are
164 // the retry policy to use, the end of the eventually consistent time window, and the backoff scaling factor
165 // If eventual consistency is not considered, this function should return the unmodified policy that was
166 // provided as input, along with (*time.Time)(nil) (no time window), and 1.0 (unscaled backoff).
167 DeterminePolicyToUse func(policy RetryPolicy) (RetryPolicy, *time.Time, float64)
168 169 // if the retry policy considers eventual consistency, but there is no eventual consistency present
170 // the retries will fall back to the policy specified here; recommendation is to set this to DefaultRetryPolicyWithoutEventualConsistency()
171 NonEventuallyConsistentPolicy *RetryPolicy
172 173 // Stores the maximum cumulative backoff in seconds. This can usually be calculated using
174 // MaximumNumberAttempts, MinSleepBetween, MaxSleepBetween, and ExponentialBackoffBase,
175 // but if MaximumNumberAttempts is 0 (unlimited attempts), then this needs to be set explicitly
176 // for Eventual Consistency retries to work.
177 MaximumCumulativeBackoffWithoutJitter float64
178 }
179 180 // GlobalRetry is user defined global level retry policy, it would impact all services, the precedence is lower
181 // than user defined client/request level retry policy
182 var GlobalRetry *RetryPolicy = nil
183 184 // RetryPolicyOption is the type of the options for NewRetryPolicy.
185 type RetryPolicyOption func(rp *RetryPolicy)
186 187 // String Converts retry policy to human-readable string representation
188 func (rp RetryPolicy) String() string {
189 return fmt.Sprintf("{MaximumNumberAttempts=%v, MinSleepBetween=%v, MaxSleepBetween=%v, ExponentialBackoffBase=%v, NonEventuallyConsistentPolicy=%v}",
190 rp.MaximumNumberAttempts, rp.MinSleepBetween, rp.MaxSleepBetween, rp.ExponentialBackoffBase, rp.NonEventuallyConsistentPolicy)
191 }
192 193 // Validate returns true if the RetryPolicy is valid; if not, it also returns an error.
194 func (rp *RetryPolicy) validate() (success bool, err error) {
195 var errorStrings []string
196 if rp.ShouldRetryOperation == nil {
197 errorStrings = append(errorStrings, "ShouldRetryOperation may not be nil")
198 }
199 if rp.NextDuration == nil {
200 errorStrings = append(errorStrings, "NextDuration may not be nil")
201 }
202 if rp.NonEventuallyConsistentPolicy != nil {
203 if rp.MaximumNumberAttempts == 0 && rp.MaximumCumulativeBackoffWithoutJitter <= 0 {
204 errorStrings = append(errorStrings, "If eventual consistency is handled, and the MaximumNumberAttempts of the EC retry policy is 0 (unlimited attempts), then the MaximumCumulativeBackoffWithoutJitter of the EC retry policy must be positive; used WithUnlimitedAttempts instead")
205 }
206 nonEcRp := rp.NonEventuallyConsistentPolicy
207 if nonEcRp.MaximumNumberAttempts == 0 && nonEcRp.MaximumCumulativeBackoffWithoutJitter <= 0 {
208 errorStrings = append(errorStrings, "If eventual consistency is handled, and the MaximumNumberAttempts of the non-EC retry policy is 0 (unlimited attempts), then the MaximumCumulativeBackoffWithoutJitter of the non-EC retry policy must be positive; used WithUnlimitedAttempts instead")
209 }
210 }
211 if len(errorStrings) > 0 {
212 return false, errors.New(strings.Join(errorStrings, ", "))
213 }
214 215 // some legacy code constructing RetryPolicy instances directly may not have set DeterminePolicyToUse.
216 // In that case, just assume that it doesn't handle eventual consistency.
217 if rp.DeterminePolicyToUse == nil {
218 rp.DeterminePolicyToUse = returnSamePolicy
219 }
220 221 return true, nil
222 }
223 224 // GetMaximumCumulativeBackoffWithoutJitter returns the maximum cumulative backoff the retry policy would do,
225 // taking into account whether eventually consistency is considered or not.
226 // This function uses either GetMaximumCumulativeBackoffWithoutJitter or GetMaximumCumulativeEventuallyConsistentBackoffWithoutJitter,
227 // whichever is appropriate
228 func (rp RetryPolicy) GetMaximumCumulativeBackoffWithoutJitter() time.Duration {
229 if rp.NonEventuallyConsistentPolicy == nil {
230 return GetMaximumCumulativeBackoffWithoutJitter(rp)
231 }
232 return GetMaximumCumulativeEventuallyConsistentBackoffWithoutJitter(rp)
233 }
234 235 //
236 // Functions to calculate backoff and maximum cumulative backoff
237 //
238 239 // GetBackoffWithoutJitter calculates the backoff without jitter for the attempt, given the retry policy.
240 func GetBackoffWithoutJitter(policy RetryPolicy, attempt uint) time.Duration {
241 return time.Duration(getBackoffWithoutJitterHelper(policy.MinSleepBetween, policy.MaxSleepBetween, policy.ExponentialBackoffBase, attempt)) * time.Second
242 }
243 244 // getBackoffWithoutJitterHelper calculates the backoff without jitter for the attempt, given the loose retry policy values.
245 func getBackoffWithoutJitterHelper(minSleepBetween float64, maxSleepBetween float64, exponentialBackoffBase float64, attempt uint) float64 {
246 sleepTime := math.Pow(exponentialBackoffBase, float64(attempt-1))
247 if sleepTime < minSleepBetween {
248 sleepTime = minSleepBetween
249 }
250 if sleepTime > maxSleepBetween {
251 sleepTime = maxSleepBetween
252 }
253 return sleepTime
254 }
255 256 // GetMaximumCumulativeBackoffWithoutJitter calculates the maximum backoff without jitter, according to the retry
257 // policy, if every retry attempt is made.
258 func GetMaximumCumulativeBackoffWithoutJitter(policy RetryPolicy) time.Duration {
259 return getMaximumCumulativeBackoffWithoutJitterHelper(policy.MinSleepBetween, policy.MaxSleepBetween, policy.ExponentialBackoffBase, policy.MaximumNumberAttempts, policy.MaximumCumulativeBackoffWithoutJitter)
260 }
261 262 func getMaximumCumulativeBackoffWithoutJitterHelper(minSleepBetween float64, maxSleepBetween float64, exponentialBackoffBase float64, MaximumNumberAttempts uint, MaximumCumulativeBackoffWithoutJitter float64) time.Duration {
263 var cumulative time.Duration = 0
264 265 if MaximumNumberAttempts == 0 {
266 // unlimited
267 return time.Duration(MaximumCumulativeBackoffWithoutJitter) * time.Second
268 }
269 270 // use a one-based counter because it's easier to think about operation retry in terms of attempt numbering
271 for currentOperationAttempt := uint(1); currentOperationAttempt < MaximumNumberAttempts; currentOperationAttempt++ {
272 cumulative += time.Duration(getBackoffWithoutJitterHelper(minSleepBetween, maxSleepBetween, exponentialBackoffBase, currentOperationAttempt)) * time.Second
273 }
274 return cumulative
275 }
276 277 //
278 // Functions to calculate backoff and maximum cumulative backoff for eventual consistency
279 //
280 281 // GetEventuallyConsistentBackoffWithoutJitter calculates the backoff without jitter for the attempt, given the retry policy
282 // and dealing with eventually consistent effects. The result is then multiplied by backoffScalingFactor.
283 func GetEventuallyConsistentBackoffWithoutJitter(policy RetryPolicy, attempt uint, backoffScalingFactor float64) time.Duration {
284 return time.Duration(getEventuallyConsistentBackoffWithoutJitterHelper(policy.MinSleepBetween, policy.MaxSleepBetween, policy.ExponentialBackoffBase, attempt, backoffScalingFactor,
285 func(minSleepBetween float64, maxSleepBetween float64, exponentialBackoffBase float64, attempt uint) float64 {
286 rp := policy.NonEventuallyConsistentPolicy
287 return getBackoffWithoutJitterHelper(rp.MinSleepBetween, rp.MaxSleepBetween, rp.ExponentialBackoffBase, attempt)
288 })*1000) * time.Millisecond
289 }
290 291 // getEventuallyConsistentBackoffWithoutJitterHelper calculates the backoff without jitter for the attempt, given the loose retry policy values,
292 // and dealing with eventually consistent effects. The result is then multiplied by backoffScalingFactor.
293 func getEventuallyConsistentBackoffWithoutJitterHelper(minSleepBetween float64, maxSleepBetween float64, exponentialBackoffBase float64, attempt uint, backoffScalingFactor float64,
294 defaultBackoffWithoutJitterHelper func(minSleepBetween float64, maxSleepBetween float64, exponentialBackoffBase float64, attempt uint) float64) float64 {
295 var sleepTime = math.Pow(exponentialBackoffBase, float64(attempt-1))
296 if sleepTime < minSleepBetween {
297 sleepTime = minSleepBetween
298 }
299 if sleepTime > maxSleepBetween {
300 sleepTime = maxSleepBetween
301 }
302 sleepTime = sleepTime * backoffScalingFactor
303 defaultSleepTime := defaultBackoffWithoutJitterHelper(minSleepBetween, maxSleepBetween, exponentialBackoffBase, attempt)
304 if defaultSleepTime > sleepTime {
305 sleepTime = defaultSleepTime
306 }
307 return sleepTime
308 }
309 310 // GetMaximumCumulativeEventuallyConsistentBackoffWithoutJitter calculates the maximum backoff without jitter, according to the retry
311 // policy and taking eventually consistent effects into account, if every retry attempt is made.
312 func GetMaximumCumulativeEventuallyConsistentBackoffWithoutJitter(policy RetryPolicy) time.Duration {
313 return getMaximumCumulativeEventuallyConsistentBackoffWithoutJitterHelper(policy.MinSleepBetween, policy.MaxSleepBetween, policy.ExponentialBackoffBase,
314 policy.MaximumNumberAttempts, policy.MaximumCumulativeBackoffWithoutJitter,
315 func(minSleepBetween float64, maxSleepBetween float64, exponentialBackoffBase float64, attempt uint) float64 {
316 rp := policy.NonEventuallyConsistentPolicy
317 return getBackoffWithoutJitterHelper(rp.MinSleepBetween, rp.MaxSleepBetween, rp.ExponentialBackoffBase, attempt)
318 })
319 }
320 321 func getMaximumCumulativeEventuallyConsistentBackoffWithoutJitterHelper(minSleepBetween float64, maxSleepBetween float64, exponentialBackoffBase float64, MaximumNumberAttempts uint,
322 MaximumCumulativeBackoffWithoutJitter float64,
323 defaultBackoffWithoutJitterHelper func(minSleepBetween float64, maxSleepBetween float64, exponentialBackoffBase float64, attempt uint) float64) time.Duration {
324 if MaximumNumberAttempts == 0 {
325 // unlimited
326 return time.Duration(MaximumCumulativeBackoffWithoutJitter) * time.Second
327 }
328 329 var cumulative time.Duration = 0
330 // use a one-based counter because it's easier to think about operation retry in terms of attempt numbering
331 for currentOperationAttempt := uint(1); currentOperationAttempt < MaximumNumberAttempts; currentOperationAttempt++ {
332 cumulative += time.Duration(getEventuallyConsistentBackoffWithoutJitterHelper(minSleepBetween, maxSleepBetween, exponentialBackoffBase, currentOperationAttempt, 1.0, defaultBackoffWithoutJitterHelper)*1000) * time.Millisecond
333 }
334 return cumulative
335 }
336 337 func returnSamePolicy(policy RetryPolicy) (RetryPolicy, *time.Time, float64) {
338 // we're returning the end of window time nonetheless, even though the default non-eventual consistency (EC)
339 // retry policy doesn't use it; this is useful in case developers wants to write an EC-aware retry policy
340 // on their own
341 eowt := EcContext.GetEndOfWindow()
342 return policy, eowt, 1.0
343 }
344 345 // NoRetryPolicy is a helper method that assembles and returns a return policy that indicates an operation should
346 // never be retried (the operation is performed exactly once).
347 func NoRetryPolicy() RetryPolicy {
348 dontRetryOperation := func(OCIOperationResponse) bool { return false }
349 zeroNextDuration := func(OCIOperationResponse) time.Duration { return 0 * time.Second }
350 return newRetryPolicyWithOptionsNoDefault(
351 WithMaximumNumberAttempts(1),
352 WithShouldRetryOperation(dontRetryOperation),
353 WithNextDuration(zeroNextDuration),
354 withMinSleepBetween(0.0*time.Second),
355 withMaxSleepBetween(0.0*time.Second),
356 withExponentialBackoffBase(0.0),
357 withDeterminePolicyToUse(returnSamePolicy),
358 withNonEventuallyConsistentPolicy(nil))
359 }
360 361 // DefaultShouldRetryOperation is the function that should be used for RetryPolicy.ShouldRetryOperation when
362 // not taking eventual consistency into account.
363 func DefaultShouldRetryOperation(r OCIOperationResponse) bool {
364 if r.Error == nil && 199 < r.Response.HTTPResponse().StatusCode && r.Response.HTTPResponse().StatusCode < 300 {
365 // success
366 return false
367 }
368 return IsErrorRetryableByDefault(r.Error)
369 }
370 371 // DefaultRetryPolicy is a helper method that assembles and returns a return policy that is defined to be a default one
372 // The default retry policy will retry on (409, IncorrectState), (429, TooManyRequests) and any 5XX errors except (501, MethodNotImplemented)
373 // The default retry behavior is using exponential backoff with jitter, the maximum wait time is 30s plus 1s jitter
374 // The maximum cumulative backoff after all 8 attempts have been made is about 1.5 minutes.
375 // It will also retry on errors affected by eventual consistency.
376 // The eventual consistency retry behavior is using exponential backoff with jitter, the maximum wait time is 45s plus 1s jitter
377 // Under eventual consistency, the maximum cumulative backoff after all 9 attempts have been made is about 4 minutes.
378 func DefaultRetryPolicy() RetryPolicy {
379 return NewRetryPolicyWithOptions(
380 ReplaceWithValuesFromRetryPolicy(DefaultRetryPolicyWithoutEventualConsistency()),
381 WithEventualConsistency())
382 }
383 384 // DefaultRetryPolicyWithoutEventualConsistency is a helper method that assembles and returns a return policy that is defined to be a default one
385 // The default retry policy will retry on (409, IncorrectState), (429, TooManyRequests) and any 5XX errors except (501, MethodNotImplemented)
386 // It will not retry on errors affected by eventual consistency.
387 // The default retry behavior is using exponential backoff with jitter, the maximum wait time is 30s plus 1s jitter
388 func DefaultRetryPolicyWithoutEventualConsistency() RetryPolicy {
389 exponentialBackoffWithJitter := func(r OCIOperationResponse) time.Duration {
390 sleepTime := getBackoffWithoutJitterHelper(defaultMinSleepBetween, defaultMaxSleepBetween, defaultExponentialBackoffBase, r.AttemptNumber)
391 nextDuration := time.Duration(1000.0*(sleepTime+rand.Float64())) * time.Millisecond
392 return nextDuration
393 }
394 return newRetryPolicyWithOptionsNoDefault(
395 WithMaximumNumberAttempts(defaultMaximumNumberAttempts),
396 WithShouldRetryOperation(DefaultShouldRetryOperation),
397 WithNextDuration(exponentialBackoffWithJitter),
398 withMinSleepBetween(defaultMinSleepBetween*time.Second),
399 withMaxSleepBetween(defaultMaxSleepBetween*time.Second),
400 withExponentialBackoffBase(defaultExponentialBackoffBase),
401 withDeterminePolicyToUse(returnSamePolicy),
402 withNonEventuallyConsistentPolicy(nil))
403 }
404 405 // EventuallyConsistentShouldRetryOperation is the function that should be used for RetryPolicy.ShouldRetryOperation when
406 // taking eventual consistency into account
407 func EventuallyConsistentShouldRetryOperation(r OCIOperationResponse) bool {
408 if r.Error == nil && 199 < r.Response.HTTPResponse().StatusCode && r.Response.HTTPResponse().StatusCode < 300 {
409 // success
410 Debugln(fmt.Sprintf("EC.ShouldRetryOperation, status = %v, 2xx, returning false", r.Response.HTTPResponse().StatusCode))
411 return false
412 }
413 if IsErrorRetryableByDefault(r.Error) {
414 return true
415 }
416 // not retryable by default
417 if _, ok := IsServiceError(r.Error); ok {
418 now := EcContext.timeNowProvider()
419 if r.EndOfWindowTime == nil || r.EndOfWindowTime.Before(now) {
420 // either no eventually consistent effects, or they have disappeared by now
421 Debugln(fmt.Sprintf("EC.ShouldRetryOperation, no EC or in the past, returning false: endOfWindowTime = %v, now = %v", r.EndOfWindowTime, now))
422 return false
423 }
424 // there were eventually consistent effects present at the time of the first request
425 // and they could still affect the retries
426 if IsErrorAffectedByEventualConsistency(r.Error) {
427 // and it's one of the three affected error codes
428 Debugln(fmt.Sprintf("EC.ShouldRetryOperation, affected by EC, EC is present: endOfWindowTime = %v, now = %v", r.EndOfWindowTime, now))
429 return true
430 }
431 return false
432 }
433 434 return false
435 }
436 437 // EventuallyConsistentRetryPolicy is a helper method that assembles and returns a return policy that is defined to be a default one
438 // plus dealing with errors affected by eventual consistency.
439 // The default retry behavior is using exponential backoff with jitter, the maximum wait time is 45s plus 1s jitter
440 func EventuallyConsistentRetryPolicy(nonEventuallyConsistentPolicy RetryPolicy) RetryPolicy {
441 if nonEventuallyConsistentPolicy.NonEventuallyConsistentPolicy != nil {
442 // already deals with eventual consistency
443 return nonEventuallyConsistentPolicy
444 }
445 exponentialBackoffWithJitter := func(r OCIOperationResponse) time.Duration {
446 sleepTime := getEventuallyConsistentBackoffWithoutJitterHelper(ecMinSleepBetween, ecMaxSleepBetween, ecExponentialBackoffBase, r.AttemptNumber, r.BackoffScalingFactor,
447 func(minSleepBetween float64, maxSleepBetween float64, exponentialBackoffBase float64, attempt uint) float64 {
448 rp := nonEventuallyConsistentPolicy
449 return getBackoffWithoutJitterHelper(rp.MinSleepBetween, rp.MaxSleepBetween, rp.ExponentialBackoffBase, attempt)
450 })
451 nextDuration := time.Duration(1000.0*(sleepTime+rand.Float64())) * time.Millisecond
452 Debugln(fmt.Sprintf("EventuallyConsistentRetryPolicy.NextDuration for attempt %v: sleepTime = %.1fs, nextDuration = %v", r.AttemptNumber, sleepTime, nextDuration))
453 return nextDuration
454 }
455 returnModifiedPolicy := func(policy RetryPolicy) (RetryPolicy, *time.Time, float64) { return determinePolicyToUse(policy) }
456 nonEventuallyConsistentPolicyCopy := newRetryPolicyWithOptionsNoDefault(
457 ReplaceWithValuesFromRetryPolicy(nonEventuallyConsistentPolicy))
458 return newRetryPolicyWithOptionsNoDefault(
459 WithMaximumNumberAttempts(ecMaximumNumberAttempts),
460 WithShouldRetryOperation(EventuallyConsistentShouldRetryOperation),
461 WithNextDuration(exponentialBackoffWithJitter),
462 withMinSleepBetween(ecMinSleepBetween*time.Second),
463 withMaxSleepBetween(ecMaxSleepBetween*time.Second),
464 withExponentialBackoffBase(ecExponentialBackoffBase),
465 withDeterminePolicyToUse(returnModifiedPolicy),
466 withNonEventuallyConsistentPolicy(&nonEventuallyConsistentPolicyCopy))
467 }
468 469 // NewRetryPolicy is a helper method for assembling a Retry Policy object. It does not handle eventual consistency, so as to not break existing code.
470 // If you want to handle eventual consistency, the simplest way to do that is to replace the code
471 //
472 // NewRetryPolicy(a, r, n)
473 //
474 // with the code
475 //
476 // NewRetryPolicyWithOptions(
477 // WithMaximumNumberAttempts(a),
478 // WithFixedBackoff(fb) // fb is the fixed backoff duration
479 // WithShouldRetryOperation(r))
480 //
481 // or
482 //
483 // NewRetryPolicyWithOptions(
484 // WithMaximumNumberAttempts(a),
485 // WithExponentialBackoff(mb, e) // mb is the maximum backoff duration, and e is the base for exponential backoff, e.g. 2.0
486 // WithShouldRetryOperation(r))
487 //
488 // or, if a == 0 (the maximum number of attempts is unlimited)
489 //
490 // NewRetryPolicyWithEventualConsistencyUnlimitedAttempts(a, r, n, mcb) // mcb is the maximum cumulative backoff duration without jitter
491 func NewRetryPolicy(attempts uint, retryOperation func(OCIOperationResponse) bool, nextDuration func(OCIOperationResponse) time.Duration) RetryPolicy {
492 return NewRetryPolicyWithOptions(
493 ReplaceWithValuesFromRetryPolicy(DefaultRetryPolicyWithoutEventualConsistency()),
494 WithMaximumNumberAttempts(attempts),
495 WithShouldRetryOperation(retryOperation),
496 WithNextDuration(nextDuration),
497 )
498 }
499 500 // NewRetryPolicyWithEventualConsistencyUnlimitedAttempts is a helper method for assembling a Retry Policy object.
501 // It does handle eventual consistency, but other than that, it is very similar to NewRetryPolicy.
502 // NewRetryPolicyWithEventualConsistency does not support limited attempts, use NewRetryPolicyWithEventualConsistency instead.
503 func NewRetryPolicyWithEventualConsistencyUnlimitedAttempts(attempts uint, retryOperation func(OCIOperationResponse) bool, nextDuration func(OCIOperationResponse) time.Duration,
504 maximumCumulativeBackoffWithoutJitter time.Duration) (*RetryPolicy, error) {
505 506 if attempts != 0 {
507 return nil, fmt.Errorf("NewRetryPolicyWithEventualConsistencyUnlimitedAttempts cannot be used with attempts != 0 (limited attempts), use NewRetryPolicyWithEventualConsistency instead")
508 }
509 510 result := NewRetryPolicyWithOptions(
511 ReplaceWithValuesFromRetryPolicy(DefaultRetryPolicyWithoutEventualConsistency()),
512 WithUnlimitedAttempts(maximumCumulativeBackoffWithoutJitter),
513 WithShouldRetryOperation(retryOperation),
514 WithNextDuration(nextDuration),
515 )
516 return &result, nil
517 }
518 519 // NewRetryPolicyWithOptions is a helper method for assembling a Retry Policy object.
520 // It starts out with the values returned by DefaultRetryPolicy() and does handle eventual consistency,
521 // unless you replace all options set using ReplaceWithValuesFromRetryPolicy(DefaultRetryPolicyWithoutEventualConsistency()).
522 func NewRetryPolicyWithOptions(opts ...RetryPolicyOption) RetryPolicy {
523 rp := &RetryPolicy{}
524 525 // start with the default retry policy
526 ReplaceWithValuesFromRetryPolicy(DefaultRetryPolicyWithoutEventualConsistency())(rp)
527 WithEventualConsistency()(rp)
528 529 // then allow changing values
530 for _, opt := range opts {
531 opt(rp)
532 }
533 534 if rp.DeterminePolicyToUse == nil {
535 rp.DeterminePolicyToUse = returnSamePolicy
536 }
537 538 return *rp
539 }
540 541 // newRetryPolicyWithOptionsNoDefault is a helper method for assembling a Retry Policy object.
542 // Contrary to newRetryPolicyWithOptions, it does not start out with the values returned by
543 // DefaultRetryPolicy().
544 func newRetryPolicyWithOptionsNoDefault(opts ...RetryPolicyOption) RetryPolicy {
545 rp := &RetryPolicy{}
546 547 // then allow changing values
548 for _, opt := range opts {
549 opt(rp)
550 }
551 552 if rp.DeterminePolicyToUse == nil {
553 rp.DeterminePolicyToUse = returnSamePolicy
554 }
555 556 return *rp
557 }
558 559 // WithMaximumNumberAttempts is the option for NewRetryPolicyWithOptions that sets the maximum number of attempts.
560 func WithMaximumNumberAttempts(attempts uint) RetryPolicyOption {
561 // this is the RetryPolicyOption function type
562 return func(rp *RetryPolicy) {
563 rp.MaximumNumberAttempts = attempts
564 }
565 }
566 567 // WithUnlimitedAttempts is the option for NewRetryPolicyWithOptions that sets unlimited number of attempts,
568 // but it needs to set a MaximumCumulativeBackoffWithoutJitter duration.
569 // If you use WithUnlimitedAttempts, you should set your own NextDuration function using WithNextDuration.
570 func WithUnlimitedAttempts(maximumCumulativeBackoffWithoutJitter time.Duration) RetryPolicyOption {
571 // this is the RetryPolicyOption function type
572 return func(rp *RetryPolicy) {
573 rp.MaximumNumberAttempts = 0
574 rp.MaximumCumulativeBackoffWithoutJitter = float64(maximumCumulativeBackoffWithoutJitter / time.Second)
575 }
576 }
577 578 // WithShouldRetryOperation is the option for NewRetryPolicyWithOptions that sets the function that checks
579 // whether retries should be performed.
580 func WithShouldRetryOperation(retryOperation func(OCIOperationResponse) bool) RetryPolicyOption {
581 // this is the RetryPolicyOption function type
582 return func(rp *RetryPolicy) {
583 rp.ShouldRetryOperation = retryOperation
584 }
585 }
586 587 // WithNextDuration is the option for NewRetryPolicyWithOptions that sets the function for computing the next
588 // backoff duration.
589 // It is preferred to use WithFixedBackoff or WithExponentialBackoff instead.
590 func WithNextDuration(nextDuration func(OCIOperationResponse) time.Duration) RetryPolicyOption {
591 // this is the RetryPolicyOption function type
592 return func(rp *RetryPolicy) {
593 rp.NextDuration = nextDuration
594 }
595 }
596 597 // withMinSleepBetween is the option for NewRetryPolicyWithOptions that sets the minimum backoff duration.
598 func withMinSleepBetween(minSleepBetween time.Duration) RetryPolicyOption {
599 // this is the RetryPolicyOption function type
600 return func(rp *RetryPolicy) {
601 rp.MinSleepBetween = float64(minSleepBetween / time.Second)
602 }
603 }
604 605 // withMaxsSleepBetween is the option for NewRetryPolicyWithOptions that sets the maximum backoff duration.
606 func withMaxSleepBetween(maxSleepBetween time.Duration) RetryPolicyOption {
607 // this is the RetryPolicyOption function type
608 return func(rp *RetryPolicy) {
609 rp.MaxSleepBetween = float64(maxSleepBetween / time.Second)
610 }
611 }
612 613 // withExponentialBackoffBase is the option for NewRetryPolicyWithOptions that sets the base for the
614 // exponential backoff
615 func withExponentialBackoffBase(base float64) RetryPolicyOption {
616 // this is the RetryPolicyOption function type
617 return func(rp *RetryPolicy) {
618 rp.ExponentialBackoffBase = base
619 }
620 }
621 622 // withDeterminePolicyToUse is the option for NewRetryPolicyWithOptions that sets the function that
623 // determines which polich should be used and if eventual consistency should be considered
624 func withDeterminePolicyToUse(determinePolicyToUse func(policy RetryPolicy) (RetryPolicy, *time.Time, float64)) RetryPolicyOption {
625 // this is the RetryPolicyOption function type
626 return func(rp *RetryPolicy) {
627 rp.DeterminePolicyToUse = determinePolicyToUse
628 }
629 }
630 631 // withNonEventuallyConsistentPolicy is the option for NewRetryPolicyWithOptions that sets the fallback
632 // strategy if eventual consistency should not be considered
633 func withNonEventuallyConsistentPolicy(nonEventuallyConsistentPolicy *RetryPolicy) RetryPolicyOption {
634 // this is the RetryPolicyOption function type
635 return func(rp *RetryPolicy) {
636 // we want a non-EC policy for NonEventuallyConsistentPolicy; make sure that NonEventuallyConsistentPolicy is nil
637 for nonEventuallyConsistentPolicy != nil && nonEventuallyConsistentPolicy.NonEventuallyConsistentPolicy != nil {
638 nonEventuallyConsistentPolicy = nonEventuallyConsistentPolicy.NonEventuallyConsistentPolicy
639 }
640 rp.NonEventuallyConsistentPolicy = nonEventuallyConsistentPolicy
641 }
642 }
643 644 // WithExponentialBackoff is an option for NewRetryPolicyWithOptions that sets the exponential backoff base,
645 // minimum and maximum sleep between attempts, and next duration function.
646 // Therefore, WithExponentialBackoff is a combination of WithNextDuration, withMinSleepBetween, withMaxSleepBetween,
647 // and withExponentialBackoffBase.
648 func WithExponentialBackoff(newMaxSleepBetween time.Duration, newExponentialBackoffBase float64) RetryPolicyOption {
649 exponentialBackoffWithJitter := func(r OCIOperationResponse) time.Duration {
650 sleepTime := getBackoffWithoutJitterHelper(defaultMinSleepBetween, newMaxSleepBetween.Seconds(), newExponentialBackoffBase, r.AttemptNumber)
651 nextDuration := time.Duration(1000.0*(sleepTime+rand.Float64())) * time.Millisecond
652 Debugln(fmt.Sprintf("NextDuration for attempt %v: sleepTime = %.1fs, nextDuration = %v", r.AttemptNumber, sleepTime, nextDuration))
653 return nextDuration
654 }
655 656 // this is the RetryPolicyOption function type
657 return func(rp *RetryPolicy) {
658 withMinSleepBetween(0)(rp)
659 withMaxSleepBetween(newMaxSleepBetween)(rp)
660 withExponentialBackoffBase(newExponentialBackoffBase)(rp)
661 WithNextDuration(exponentialBackoffWithJitter)(rp)
662 }
663 }
664 665 // WithFixedBackoff is an option for NewRetryPolicyWithOptions that sets the backoff to always be exactly the same value. There is no jitter either.
666 // Therefore, WithFixedBackoff is a combination of WithNextDuration, withMinSleepBetween, withMaxSleepBetween, and withExponentialBackoffBase.
667 func WithFixedBackoff(newSleepBetween time.Duration) RetryPolicyOption {
668 fixedBackoffWithoutJitter := func(r OCIOperationResponse) time.Duration {
669 nextDuration := newSleepBetween
670 Debugln(fmt.Sprintf("NextDuration for attempt %v: nextDuration = %v", r.AttemptNumber, nextDuration))
671 return nextDuration
672 }
673 674 // this is the RetryPolicyOption function type
675 return func(rp *RetryPolicy) {
676 withMinSleepBetween(newSleepBetween)(rp)
677 withMaxSleepBetween(newSleepBetween)(rp)
678 withExponentialBackoffBase(1.0)(rp)
679 WithNextDuration(fixedBackoffWithoutJitter)(rp)
680 }
681 }
682 683 // WithEventualConsistency is the option for NewRetryPolicyWithOptions that enables considering eventual backoff for the policy.
684 func WithEventualConsistency() RetryPolicyOption {
685 // this is the RetryPolicyOption function type
686 return func(rp *RetryPolicy) {
687 copy := RetryPolicy{
688 MaximumNumberAttempts: rp.MaximumNumberAttempts,
689 ShouldRetryOperation: rp.ShouldRetryOperation,
690 NextDuration: rp.NextDuration,
691 MinSleepBetween: rp.MinSleepBetween,
692 MaxSleepBetween: rp.MaxSleepBetween,
693 ExponentialBackoffBase: rp.ExponentialBackoffBase,
694 DeterminePolicyToUse: rp.DeterminePolicyToUse,
695 NonEventuallyConsistentPolicy: rp.NonEventuallyConsistentPolicy,
696 }
697 ecrp := EventuallyConsistentRetryPolicy(copy)
698 rp.MaximumNumberAttempts = ecrp.MaximumNumberAttempts
699 rp.ShouldRetryOperation = ecrp.ShouldRetryOperation
700 rp.NextDuration = ecrp.NextDuration
701 rp.MinSleepBetween = ecrp.MinSleepBetween
702 rp.MaxSleepBetween = ecrp.MaxSleepBetween
703 rp.ExponentialBackoffBase = ecrp.ExponentialBackoffBase
704 rp.DeterminePolicyToUse = ecrp.DeterminePolicyToUse
705 rp.NonEventuallyConsistentPolicy = ecrp.NonEventuallyConsistentPolicy
706 }
707 }
708 709 // WithConditionalOption is an option for NewRetryPolicyWithOptions that enables or disables another option.
710 func WithConditionalOption(enabled bool, otherOption RetryPolicyOption) RetryPolicyOption {
711 // this is the RetryPolicyOption function type
712 return func(rp *RetryPolicy) {
713 if enabled {
714 otherOption(rp)
715 }
716 }
717 }
718 719 // ReplaceWithValuesFromRetryPolicy is an option for NewRetryPolicyWithOptions that copies over all settings from another RetryPolicy
720 func ReplaceWithValuesFromRetryPolicy(other RetryPolicy) RetryPolicyOption {
721 // this is the RetryPolicyOption function type
722 return func(rp *RetryPolicy) {
723 rp.MaximumNumberAttempts = other.MaximumNumberAttempts
724 rp.ShouldRetryOperation = other.ShouldRetryOperation
725 rp.NextDuration = other.NextDuration
726 rp.MinSleepBetween = other.MinSleepBetween
727 rp.MaxSleepBetween = other.MaxSleepBetween
728 rp.ExponentialBackoffBase = other.ExponentialBackoffBase
729 rp.DeterminePolicyToUse = other.DeterminePolicyToUse
730 rp.NonEventuallyConsistentPolicy = other.NonEventuallyConsistentPolicy
731 rp.MaximumCumulativeBackoffWithoutJitter = other.MaximumCumulativeBackoffWithoutJitter
732 }
733 }
734 735 // shouldContinueIssuingRequests returns true if we should continue retrying a request, based on the current attempt
736 // number and the maximum number of attempts specified, or false otherwise.
737 func shouldContinueIssuingRequests(current, maximum uint) bool {
738 return maximum == UnlimitedNumAttemptsValue || current <= maximum
739 }
740 741 // RetryToken generates a retry token that must be included on any request passed to the Retry method.
742 func RetryToken() string {
743 alphanumericChars := []rune("abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ")
744 retryToken := make([]rune, generatedRetryTokenLength)
745 for i := range retryToken {
746 retryToken[i] = alphanumericChars[rand.Intn(len(alphanumericChars))]
747 }
748 return string(retryToken)
749 }
750 751 func determinePolicyToUse(policy RetryPolicy) (RetryPolicy, *time.Time, float64) {
752 initialAttemptTime := EcContext.timeNowProvider()
753 var useDefaultTimingInstead = true
754 var endOfWindowTime = (*time.Time)(nil)
755 var backoffScalingFactor = 1.0
756 var policyToUse = policy
757 758 eowt := EcContext.GetEndOfWindow()
759 if eowt != nil {
760 // there was an eventually consistent request
761 if eowt.After(initialAttemptTime) {
762 // and the eventually consistent effects may still be present
763 endOfWindowTime = eowt
764 // if the time between now and the end of the window is less than the time we normally would retry, use the default timing
765 durationToEndOfWindow := endOfWindowTime.Sub(initialAttemptTime)
766 maxCumulativeBackoffWithoutJitter := GetMaximumCumulativeBackoffWithoutJitter(*policy.NonEventuallyConsistentPolicy)
767 Debugln(fmt.Sprintf("durationToEndOfWindow = %v, maxCumulativeBackoffWithoutJitter = %v", durationToEndOfWindow, maxCumulativeBackoffWithoutJitter))
768 if durationToEndOfWindow > maxCumulativeBackoffWithoutJitter {
769 // the end of the eventually consistent window is later than when default retries would end
770 // do not use default timing
771 maximumCumulativeBackoffWithoutJitter := GetMaximumCumulativeEventuallyConsistentBackoffWithoutJitter(policy)
772 backoffScalingFactor = float64(durationToEndOfWindow) / float64(maximumCumulativeBackoffWithoutJitter)
773 useDefaultTimingInstead = false
774 Debugln(fmt.Sprintf("Use eventually consistent timing, durationToEndOfWindow = %v, maximumCumulativeBackoffWithoutJitter = %v, backoffScalingFactor = %.2f",
775 durationToEndOfWindow, maximumCumulativeBackoffWithoutJitter, backoffScalingFactor))
776 } else {
777 Debugln("Use default timing, end of EC window is sooner than default retries")
778 }
779 } else {
780 useDefaultTimingInstead = false
781 policyToUse = *policy.NonEventuallyConsistentPolicy
782 Debugln("Use default timing and strategy, end of EC window is in the past")
783 }
784 } else {
785 useDefaultTimingInstead = false
786 policyToUse = *policy.NonEventuallyConsistentPolicy
787 Debugln("Use default timing and strategy, no EC window set")
788 }
789 790 if useDefaultTimingInstead {
791 // use timing from defaultRetryPolicy, but whether to retry from the policy that was passed into this request
792 policyToUse = NewRetryPolicyWithOptions(
793 ReplaceWithValuesFromRetryPolicy(*policy.NonEventuallyConsistentPolicy),
794 WithShouldRetryOperation(policy.ShouldRetryOperation))
795 }
796 797 return policyToUse, endOfWindowTime, backoffScalingFactor
798 }
799 800 // Retry is a package-level operation that executes the retryable request using the specified operation and retry policy.
801 func Retry(ctx context.Context, request OCIRetryableRequest, operation OCIOperation, policy RetryPolicy) (OCIResponse, error) {
802 type retrierResult struct {
803 response OCIResponse
804 err error
805 }
806 807 var response OCIResponse
808 var err error
809 retrierChannel := make(chan retrierResult, 1)
810 811 validated, validateError := policy.validate()
812 if !validated {
813 return nil, validateError
814 }
815 816 initialAttemptTime := time.Now()
817 818 go func() {
819 // Deal with panics more graciously
820 defer func() {
821 if r := recover(); r != nil {
822 stackBuffer := make([]byte, 1024)
823 bytesWritten := runtime.Stack(stackBuffer, false)
824 stack := string(stackBuffer[:bytesWritten])
825 error := fmt.Errorf("panicked while retrying operation. Panic was: %s\nStack: %s", r, stack)
826 Debugln(error)
827 retrierChannel <- retrierResult{nil, error}
828 }
829 }()
830 // if request body is binary request body and seekable, save the current position
831 var curPos int64 = 0
832 isSeekable := false
833 rsc, isBinaryRequest := request.BinaryRequestBody()
834 if rsc != nil && rsc.rc != nil {
835 defer rsc.rc.Close()
836 }
837 if policy.MaximumNumberAttempts != uint(1) {
838 if rsc.Seekable() {
839 isSeekable = true
840 curPos, _ = rsc.Seek(0, io.SeekCurrent)
841 }
842 }
843 844 // some legacy code constructing RetryPolicy instances directly may not have set DeterminePolicyToUse.
845 // In that case, just assume that it doesn't handle eventual consistency.
846 if policy.DeterminePolicyToUse == nil {
847 policy.DeterminePolicyToUse = returnSamePolicy
848 }
849 850 // this determines which policy to use, when the eventual consistency window ends, and what the backoff
851 // scaling factor should be
852 policyToUse, endOfWindowTime, backoffScalingFactor := policy.DeterminePolicyToUse(policy)
853 Debugln(fmt.Sprintf("Retry policy to use: %v", policyToUse))
854 retryStartTime := time.Now()
855 extraHeaders := make(map[string]string)
856 857 if policy.MaximumNumberAttempts == 1 {
858 extraHeaders[requestHeaderOpcClientRetries] = "false"
859 } else {
860 extraHeaders[requestHeaderOpcClientRetries] = "true"
861 }
862 863 // use a one-based counter because it's easier to think about operation retry in terms of attempt numbering
864 for currentOperationAttempt := uint(1); shouldContinueIssuingRequests(currentOperationAttempt, policyToUse.MaximumNumberAttempts); currentOperationAttempt++ {
865 Debugln(fmt.Sprintf("operation attempt #%v", currentOperationAttempt))
866 // rewind body once needed
867 if isSeekable {
868 rsc = NewOCIReadSeekCloser(rsc.rc)
869 rsc.Seek(curPos, io.SeekStart)
870 }
871 response, err = operation(ctx, request, rsc, extraHeaders)
872 873 operationResponse := NewOCIOperationResponseExtended(response, err, currentOperationAttempt, endOfWindowTime, backoffScalingFactor, initialAttemptTime)
874 875 if !policyToUse.ShouldRetryOperation(operationResponse) {
876 // we should NOT retry operation based on response and/or error => return
877 retrierChannel <- retrierResult{response, err}
878 return
879 }
880 881 // if the request body type is stream, requested retry but doesn't resettable, throw error and stop retrying
882 if isBinaryRequest && !isSeekable {
883 retrierChannel <- retrierResult{response, NonSeekableRequestRetryFailure{err}}
884 return
885 }
886 887 duration := policyToUse.NextDuration(operationResponse)
888 //The following condition is kept for backwards compatibility reasons
889 if deadline, ok := ctx.Deadline(); ok && EcContext.timeNowProvider().Add(duration).After(deadline) {
890 // we want to retry the operation, but the policy is telling us to wait for a duration that exceeds
891 // the specified overall deadline for the operation => instead of waiting for however long that
892 // time period is and then aborting, abort now and save the cycles
893 retrierChannel <- retrierResult{response, DeadlineExceededByBackoff}
894 return
895 }
896 Debugln(fmt.Sprintf("waiting %v before retrying operation", duration))
897 // sleep before retrying the operation
898 <-time.After(duration)
899 }
900 retryEndTime := time.Now()
901 Debugln(fmt.Sprintf("Total Latency for this API call is: %v ms", retryEndTime.Sub(retryStartTime).Milliseconds()))
902 retrierChannel <- retrierResult{response, err}
903 }()
904 905 select {
906 case <-ctx.Done():
907 return response, ctx.Err()
908 case result := <-retrierChannel:
909 return result.response, result.err
910 }
911 }
912