exponential_backoff.go 4.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122
  1. /*
  2. Copyright 2016 The Kubernetes Authors.
  3. Licensed under the Apache License, Version 2.0 (the "License");
  4. you may not use this file except in compliance with the License.
  5. You may obtain a copy of the License at
  6. http://www.apache.org/licenses/LICENSE-2.0
  7. Unless required by applicable law or agreed to in writing, software
  8. distributed under the License is distributed on an "AS IS" BASIS,
  9. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  10. See the License for the specific language governing permissions and
  11. limitations under the License.
  12. */
  13. // Package exponentialbackoff contains logic for implementing exponential
  14. // backoff for GoRoutineMap and NestedPendingOperations.
  15. package exponentialbackoff
  16. import (
  17. "fmt"
  18. "time"
  19. )
  20. const (
  21. // initialDurationBeforeRetry is the amount of time after an error occurs
  22. // that GoroutineMap will refuse to allow another operation to start with
  23. // the same target (if exponentialBackOffOnError is enabled). Each
  24. // successive error results in a wait 2x times the previous.
  25. initialDurationBeforeRetry time.Duration = 500 * time.Millisecond
  26. // maxDurationBeforeRetry is the maximum amount of time that
  27. // durationBeforeRetry will grow to due to exponential backoff.
  28. // Value is slightly offset from 2 minutes to make timeouts due to this
  29. // constant recognizable.
  30. maxDurationBeforeRetry time.Duration = 2*time.Minute + 2*time.Second
  31. )
  32. // ExponentialBackoff contains the last occurrence of an error and the duration
  33. // that retries are not permitted.
  34. type ExponentialBackoff struct {
  35. lastError error
  36. lastErrorTime time.Time
  37. durationBeforeRetry time.Duration
  38. }
  39. // SafeToRetry returns an error if the durationBeforeRetry period for the given
  40. // lastErrorTime has not yet expired. Otherwise it returns nil.
  41. func (expBackoff *ExponentialBackoff) SafeToRetry(operationName string) error {
  42. if time.Since(expBackoff.lastErrorTime) <= expBackoff.durationBeforeRetry {
  43. return NewExponentialBackoffError(operationName, *expBackoff)
  44. }
  45. return nil
  46. }
  47. func (expBackoff *ExponentialBackoff) Update(err *error) {
  48. if expBackoff.durationBeforeRetry == 0 {
  49. expBackoff.durationBeforeRetry = initialDurationBeforeRetry
  50. } else {
  51. expBackoff.durationBeforeRetry = 2 * expBackoff.durationBeforeRetry
  52. if expBackoff.durationBeforeRetry > maxDurationBeforeRetry {
  53. expBackoff.durationBeforeRetry = maxDurationBeforeRetry
  54. }
  55. }
  56. expBackoff.lastError = *err
  57. expBackoff.lastErrorTime = time.Now()
  58. }
  59. func (expBackoff *ExponentialBackoff) GenerateNoRetriesPermittedMsg(operationName string) string {
  60. return fmt.Sprintf("Operation for %q failed. No retries permitted until %v (durationBeforeRetry %v). Error: %q",
  61. operationName,
  62. expBackoff.lastErrorTime.Add(expBackoff.durationBeforeRetry),
  63. expBackoff.durationBeforeRetry,
  64. expBackoff.lastError)
  65. }
  66. // NewExponentialBackoffError returns a new instance of ExponentialBackoff error.
  67. func NewExponentialBackoffError(
  68. operationName string, expBackoff ExponentialBackoff) error {
  69. return exponentialBackoffError{
  70. operationName: operationName,
  71. expBackoff: expBackoff,
  72. }
  73. }
  74. // IsExponentialBackoff returns true if an error returned from GoroutineMap
  75. // indicates that a new operation can not be started because
  76. // exponentialBackOffOnError is enabled and a previous operation with the same
  77. // operation failed within the durationBeforeRetry period.
  78. func IsExponentialBackoff(err error) bool {
  79. switch err.(type) {
  80. case exponentialBackoffError:
  81. return true
  82. default:
  83. return false
  84. }
  85. }
  86. // exponentialBackoffError is the error returned returned from GoroutineMap when
  87. // a new operation can not be started because exponentialBackOffOnError is
  88. // enabled and a previous operation with the same operation failed within the
  89. // durationBeforeRetry period.
  90. type exponentialBackoffError struct {
  91. operationName string
  92. expBackoff ExponentialBackoff
  93. }
  94. var _ error = exponentialBackoffError{}
  95. func (err exponentialBackoffError) Error() string {
  96. return fmt.Sprintf(
  97. "Failed to create operation with name %q. An operation with that name failed at %v. No retries permitted until %v (%v). Last error: %q.",
  98. err.operationName,
  99. err.expBackoff.lastErrorTime,
  100. err.expBackoff.lastErrorTime.Add(err.expBackoff.durationBeforeRetry),
  101. err.expBackoff.durationBeforeRetry,
  102. err.expBackoff.lastError)
  103. }