sender.go 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412
  1. package autorest
  2. // Copyright 2017 Microsoft Corporation
  3. //
  4. // Licensed under the Apache License, Version 2.0 (the "License");
  5. // you may not use this file except in compliance with the License.
  6. // You may obtain a copy of the License at
  7. //
  8. // http://www.apache.org/licenses/LICENSE-2.0
  9. //
  10. // Unless required by applicable law or agreed to in writing, software
  11. // distributed under the License is distributed on an "AS IS" BASIS,
  12. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13. // See the License for the specific language governing permissions and
  14. // limitations under the License.
  15. import (
  16. "context"
  17. "crypto/tls"
  18. "fmt"
  19. "log"
  20. "math"
  21. "net/http"
  22. "net/http/cookiejar"
  23. "strconv"
  24. "time"
  25. "github.com/Azure/go-autorest/tracing"
  26. )
  27. // used as a key type in context.WithValue()
  28. type ctxSendDecorators struct{}
  29. // WithSendDecorators adds the specified SendDecorators to the provided context.
  30. // If no SendDecorators are provided the context is unchanged.
  31. func WithSendDecorators(ctx context.Context, sendDecorator []SendDecorator) context.Context {
  32. if len(sendDecorator) == 0 {
  33. return ctx
  34. }
  35. return context.WithValue(ctx, ctxSendDecorators{}, sendDecorator)
  36. }
  37. // GetSendDecorators returns the SendDecorators in the provided context or the provided default SendDecorators.
  38. func GetSendDecorators(ctx context.Context, defaultSendDecorators ...SendDecorator) []SendDecorator {
  39. inCtx := ctx.Value(ctxSendDecorators{})
  40. if sd, ok := inCtx.([]SendDecorator); ok {
  41. return sd
  42. }
  43. return defaultSendDecorators
  44. }
  45. // Sender is the interface that wraps the Do method to send HTTP requests.
  46. //
  47. // The standard http.Client conforms to this interface.
  48. type Sender interface {
  49. Do(*http.Request) (*http.Response, error)
  50. }
  51. // SenderFunc is a method that implements the Sender interface.
  52. type SenderFunc func(*http.Request) (*http.Response, error)
  53. // Do implements the Sender interface on SenderFunc.
  54. func (sf SenderFunc) Do(r *http.Request) (*http.Response, error) {
  55. return sf(r)
  56. }
  57. // SendDecorator takes and possibly decorates, by wrapping, a Sender. Decorators may affect the
  58. // http.Request and pass it along or, first, pass the http.Request along then react to the
  59. // http.Response result.
  60. type SendDecorator func(Sender) Sender
  61. // CreateSender creates, decorates, and returns, as a Sender, the default http.Client.
  62. func CreateSender(decorators ...SendDecorator) Sender {
  63. return DecorateSender(sender(tls.RenegotiateNever), decorators...)
  64. }
  65. // DecorateSender accepts a Sender and a, possibly empty, set of SendDecorators, which is applies to
  66. // the Sender. Decorators are applied in the order received, but their affect upon the request
  67. // depends on whether they are a pre-decorator (change the http.Request and then pass it along) or a
  68. // post-decorator (pass the http.Request along and react to the results in http.Response).
  69. func DecorateSender(s Sender, decorators ...SendDecorator) Sender {
  70. for _, decorate := range decorators {
  71. s = decorate(s)
  72. }
  73. return s
  74. }
  75. // Send sends, by means of the default http.Client, the passed http.Request, returning the
  76. // http.Response and possible error. It also accepts a, possibly empty, set of SendDecorators which
  77. // it will apply the http.Client before invoking the Do method.
  78. //
  79. // Send is a convenience method and not recommended for production. Advanced users should use
  80. // SendWithSender, passing and sharing their own Sender (e.g., instance of http.Client).
  81. //
  82. // Send will not poll or retry requests.
  83. func Send(r *http.Request, decorators ...SendDecorator) (*http.Response, error) {
  84. return SendWithSender(sender(tls.RenegotiateNever), r, decorators...)
  85. }
  86. // SendWithSender sends the passed http.Request, through the provided Sender, returning the
  87. // http.Response and possible error. It also accepts a, possibly empty, set of SendDecorators which
  88. // it will apply the http.Client before invoking the Do method.
  89. //
  90. // SendWithSender will not poll or retry requests.
  91. func SendWithSender(s Sender, r *http.Request, decorators ...SendDecorator) (*http.Response, error) {
  92. return DecorateSender(s, decorators...).Do(r)
  93. }
  94. func sender(renengotiation tls.RenegotiationSupport) Sender {
  95. // Use behaviour compatible with DefaultTransport, but require TLS minimum version.
  96. defaultTransport := http.DefaultTransport.(*http.Transport)
  97. transport := &http.Transport{
  98. Proxy: defaultTransport.Proxy,
  99. DialContext: defaultTransport.DialContext,
  100. MaxIdleConns: defaultTransport.MaxIdleConns,
  101. IdleConnTimeout: defaultTransport.IdleConnTimeout,
  102. TLSHandshakeTimeout: defaultTransport.TLSHandshakeTimeout,
  103. ExpectContinueTimeout: defaultTransport.ExpectContinueTimeout,
  104. TLSClientConfig: &tls.Config{
  105. MinVersion: tls.VersionTLS12,
  106. Renegotiation: renengotiation,
  107. },
  108. }
  109. var roundTripper http.RoundTripper = transport
  110. if tracing.IsEnabled() {
  111. roundTripper = tracing.NewTransport(transport)
  112. }
  113. j, _ := cookiejar.New(nil)
  114. return &http.Client{Jar: j, Transport: roundTripper}
  115. }
  116. // AfterDelay returns a SendDecorator that delays for the passed time.Duration before
  117. // invoking the Sender. The delay may be terminated by closing the optional channel on the
  118. // http.Request. If canceled, no further Senders are invoked.
  119. func AfterDelay(d time.Duration) SendDecorator {
  120. return func(s Sender) Sender {
  121. return SenderFunc(func(r *http.Request) (*http.Response, error) {
  122. if !DelayForBackoff(d, 0, r.Context().Done()) {
  123. return nil, fmt.Errorf("autorest: AfterDelay canceled before full delay")
  124. }
  125. return s.Do(r)
  126. })
  127. }
  128. }
  129. // AsIs returns a SendDecorator that invokes the passed Sender without modifying the http.Request.
  130. func AsIs() SendDecorator {
  131. return func(s Sender) Sender {
  132. return SenderFunc(func(r *http.Request) (*http.Response, error) {
  133. return s.Do(r)
  134. })
  135. }
  136. }
  137. // DoCloseIfError returns a SendDecorator that first invokes the passed Sender after which
  138. // it closes the response if the passed Sender returns an error and the response body exists.
  139. func DoCloseIfError() SendDecorator {
  140. return func(s Sender) Sender {
  141. return SenderFunc(func(r *http.Request) (*http.Response, error) {
  142. resp, err := s.Do(r)
  143. if err != nil {
  144. Respond(resp, ByDiscardingBody(), ByClosing())
  145. }
  146. return resp, err
  147. })
  148. }
  149. }
  150. // DoErrorIfStatusCode returns a SendDecorator that emits an error if the response StatusCode is
  151. // among the set passed. Since these are artificial errors, the response body may still require
  152. // closing.
  153. func DoErrorIfStatusCode(codes ...int) SendDecorator {
  154. return func(s Sender) Sender {
  155. return SenderFunc(func(r *http.Request) (*http.Response, error) {
  156. resp, err := s.Do(r)
  157. if err == nil && ResponseHasStatusCode(resp, codes...) {
  158. err = NewErrorWithResponse("autorest", "DoErrorIfStatusCode", resp, "%v %v failed with %s",
  159. resp.Request.Method,
  160. resp.Request.URL,
  161. resp.Status)
  162. }
  163. return resp, err
  164. })
  165. }
  166. }
  167. // DoErrorUnlessStatusCode returns a SendDecorator that emits an error unless the response
  168. // StatusCode is among the set passed. Since these are artificial errors, the response body
  169. // may still require closing.
  170. func DoErrorUnlessStatusCode(codes ...int) SendDecorator {
  171. return func(s Sender) Sender {
  172. return SenderFunc(func(r *http.Request) (*http.Response, error) {
  173. resp, err := s.Do(r)
  174. if err == nil && !ResponseHasStatusCode(resp, codes...) {
  175. err = NewErrorWithResponse("autorest", "DoErrorUnlessStatusCode", resp, "%v %v failed with %s",
  176. resp.Request.Method,
  177. resp.Request.URL,
  178. resp.Status)
  179. }
  180. return resp, err
  181. })
  182. }
  183. }
  184. // DoPollForStatusCodes returns a SendDecorator that polls if the http.Response contains one of the
  185. // passed status codes. It expects the http.Response to contain a Location header providing the
  186. // URL at which to poll (using GET) and will poll until the time passed is equal to or greater than
  187. // the supplied duration. It will delay between requests for the duration specified in the
  188. // RetryAfter header or, if the header is absent, the passed delay. Polling may be canceled by
  189. // closing the optional channel on the http.Request.
  190. func DoPollForStatusCodes(duration time.Duration, delay time.Duration, codes ...int) SendDecorator {
  191. return func(s Sender) Sender {
  192. return SenderFunc(func(r *http.Request) (resp *http.Response, err error) {
  193. resp, err = s.Do(r)
  194. if err == nil && ResponseHasStatusCode(resp, codes...) {
  195. r, err = NewPollingRequestWithContext(r.Context(), resp)
  196. for err == nil && ResponseHasStatusCode(resp, codes...) {
  197. Respond(resp,
  198. ByDiscardingBody(),
  199. ByClosing())
  200. resp, err = SendWithSender(s, r,
  201. AfterDelay(GetRetryAfter(resp, delay)))
  202. }
  203. }
  204. return resp, err
  205. })
  206. }
  207. }
  208. // DoRetryForAttempts returns a SendDecorator that retries a failed request for up to the specified
  209. // number of attempts, exponentially backing off between requests using the supplied backoff
  210. // time.Duration (which may be zero). Retrying may be canceled by closing the optional channel on
  211. // the http.Request.
  212. func DoRetryForAttempts(attempts int, backoff time.Duration) SendDecorator {
  213. return func(s Sender) Sender {
  214. return SenderFunc(func(r *http.Request) (resp *http.Response, err error) {
  215. rr := NewRetriableRequest(r)
  216. for attempt := 0; attempt < attempts; attempt++ {
  217. err = rr.Prepare()
  218. if err != nil {
  219. return resp, err
  220. }
  221. resp, err = s.Do(rr.Request())
  222. if err == nil {
  223. return resp, err
  224. }
  225. if !DelayForBackoff(backoff, attempt, r.Context().Done()) {
  226. return nil, r.Context().Err()
  227. }
  228. }
  229. return resp, err
  230. })
  231. }
  232. }
  233. // DoRetryForStatusCodes returns a SendDecorator that retries for specified statusCodes for up to the specified
  234. // number of attempts, exponentially backing off between requests using the supplied backoff
  235. // time.Duration (which may be zero). Retrying may be canceled by cancelling the context on the http.Request.
  236. // NOTE: Code http.StatusTooManyRequests (429) will *not* be counted against the number of attempts.
  237. func DoRetryForStatusCodes(attempts int, backoff time.Duration, codes ...int) SendDecorator {
  238. return func(s Sender) Sender {
  239. return SenderFunc(func(r *http.Request) (*http.Response, error) {
  240. return doRetryForStatusCodesImpl(s, r, false, attempts, backoff, 0, codes...)
  241. })
  242. }
  243. }
  244. // DoRetryForStatusCodesWithCap returns a SendDecorator that retries for specified statusCodes for up to the
  245. // specified number of attempts, exponentially backing off between requests using the supplied backoff
  246. // time.Duration (which may be zero). To cap the maximum possible delay between iterations specify a value greater
  247. // than zero for cap. Retrying may be canceled by cancelling the context on the http.Request.
  248. func DoRetryForStatusCodesWithCap(attempts int, backoff, cap time.Duration, codes ...int) SendDecorator {
  249. return func(s Sender) Sender {
  250. return SenderFunc(func(r *http.Request) (*http.Response, error) {
  251. return doRetryForStatusCodesImpl(s, r, true, attempts, backoff, cap, codes...)
  252. })
  253. }
  254. }
  255. func doRetryForStatusCodesImpl(s Sender, r *http.Request, count429 bool, attempts int, backoff, cap time.Duration, codes ...int) (resp *http.Response, err error) {
  256. rr := NewRetriableRequest(r)
  257. // Increment to add the first call (attempts denotes number of retries)
  258. for attempt := 0; attempt < attempts+1; {
  259. err = rr.Prepare()
  260. if err != nil {
  261. return
  262. }
  263. resp, err = s.Do(rr.Request())
  264. // if the error isn't temporary don't bother retrying
  265. if err != nil && !IsTemporaryNetworkError(err) {
  266. return
  267. }
  268. // we want to retry if err is not nil (e.g. transient network failure). note that for failed authentication
  269. // resp and err will both have a value, so in this case we don't want to retry as it will never succeed.
  270. if err == nil && !ResponseHasStatusCode(resp, codes...) || IsTokenRefreshError(err) {
  271. return resp, err
  272. }
  273. delayed := DelayWithRetryAfter(resp, r.Context().Done())
  274. if !delayed && !DelayForBackoffWithCap(backoff, cap, attempt, r.Context().Done()) {
  275. return resp, r.Context().Err()
  276. }
  277. // when count429 == false don't count a 429 against the number
  278. // of attempts so that we continue to retry until it succeeds
  279. if count429 || (resp == nil || resp.StatusCode != http.StatusTooManyRequests) {
  280. attempt++
  281. }
  282. }
  283. return resp, err
  284. }
  285. // DelayWithRetryAfter invokes time.After for the duration specified in the "Retry-After" header.
  286. // The value of Retry-After can be either the number of seconds or a date in RFC1123 format.
  287. // The function returns true after successfully waiting for the specified duration. If there is
  288. // no Retry-After header or the wait is cancelled the return value is false.
  289. func DelayWithRetryAfter(resp *http.Response, cancel <-chan struct{}) bool {
  290. if resp == nil {
  291. return false
  292. }
  293. var dur time.Duration
  294. ra := resp.Header.Get("Retry-After")
  295. if retryAfter, _ := strconv.Atoi(ra); retryAfter > 0 {
  296. dur = time.Duration(retryAfter) * time.Second
  297. } else if t, err := time.Parse(time.RFC1123, ra); err == nil {
  298. dur = t.Sub(time.Now())
  299. }
  300. if dur > 0 {
  301. select {
  302. case <-time.After(dur):
  303. return true
  304. case <-cancel:
  305. return false
  306. }
  307. }
  308. return false
  309. }
  310. // DoRetryForDuration returns a SendDecorator that retries the request until the total time is equal
  311. // to or greater than the specified duration, exponentially backing off between requests using the
  312. // supplied backoff time.Duration (which may be zero). Retrying may be canceled by closing the
  313. // optional channel on the http.Request.
  314. func DoRetryForDuration(d time.Duration, backoff time.Duration) SendDecorator {
  315. return func(s Sender) Sender {
  316. return SenderFunc(func(r *http.Request) (resp *http.Response, err error) {
  317. rr := NewRetriableRequest(r)
  318. end := time.Now().Add(d)
  319. for attempt := 0; time.Now().Before(end); attempt++ {
  320. err = rr.Prepare()
  321. if err != nil {
  322. return resp, err
  323. }
  324. resp, err = s.Do(rr.Request())
  325. if err == nil {
  326. return resp, err
  327. }
  328. if !DelayForBackoff(backoff, attempt, r.Context().Done()) {
  329. return nil, r.Context().Err()
  330. }
  331. }
  332. return resp, err
  333. })
  334. }
  335. }
  336. // WithLogging returns a SendDecorator that implements simple before and after logging of the
  337. // request.
  338. func WithLogging(logger *log.Logger) SendDecorator {
  339. return func(s Sender) Sender {
  340. return SenderFunc(func(r *http.Request) (*http.Response, error) {
  341. logger.Printf("Sending %s %s", r.Method, r.URL)
  342. resp, err := s.Do(r)
  343. if err != nil {
  344. logger.Printf("%s %s received error '%v'", r.Method, r.URL, err)
  345. } else {
  346. logger.Printf("%s %s received %s", r.Method, r.URL, resp.Status)
  347. }
  348. return resp, err
  349. })
  350. }
  351. }
  352. // DelayForBackoff invokes time.After for the supplied backoff duration raised to the power of
  353. // passed attempt (i.e., an exponential backoff delay). Backoff duration is in seconds and can set
  354. // to zero for no delay. The delay may be canceled by closing the passed channel. If terminated early,
  355. // returns false.
  356. // Note: Passing attempt 1 will result in doubling "backoff" duration. Treat this as a zero-based attempt
  357. // count.
  358. func DelayForBackoff(backoff time.Duration, attempt int, cancel <-chan struct{}) bool {
  359. return DelayForBackoffWithCap(backoff, 0, attempt, cancel)
  360. }
  361. // DelayForBackoffWithCap invokes time.After for the supplied backoff duration raised to the power of
  362. // passed attempt (i.e., an exponential backoff delay). Backoff duration is in seconds and can set
  363. // to zero for no delay. To cap the maximum possible delay specify a value greater than zero for cap.
  364. // The delay may be canceled by closing the passed channel. If terminated early, returns false.
  365. // Note: Passing attempt 1 will result in doubling "backoff" duration. Treat this as a zero-based attempt
  366. // count.
  367. func DelayForBackoffWithCap(backoff, cap time.Duration, attempt int, cancel <-chan struct{}) bool {
  368. d := time.Duration(backoff.Seconds()*math.Pow(2, float64(attempt))) * time.Second
  369. if cap > 0 && d > cap {
  370. d = cap
  371. }
  372. select {
  373. case <-time.After(d):
  374. return true
  375. case <-cancel:
  376. return false
  377. }
  378. }