1 // Copyright 2009 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
4 5 // HTTP client. See RFC 7230 through 7235.
6 //
7 // This is the high-level Client interface.
8 // The low-level implementation is in transport.go.
9 10 package http
11 12 import (
13 "context"
14 "crypto/tls"
15 "encoding/base64"
16 "errors"
17 "fmt"
18 "io"
19 "log"
20 "net/http/internal/ascii"
21 "net/url"
22 23 "slices"
24 "bytes"
25 "sync"
26 "sync/atomic"
27 "time"
28 )
29 30 // A Client is an HTTP client. Its zero value ([DefaultClient]) is a
31 // usable client that uses [DefaultTransport].
32 //
33 // The [Client.Transport] typically has internal state (cached TCP
34 // connections), so Clients should be reused instead of created as
35 // needed. Clients are safe for concurrent use by multiple goroutines.
36 //
37 // A Client is higher-level than a [RoundTripper] (such as [Transport])
38 // and additionally handles HTTP details such as cookies and
39 // redirects.
40 //
41 // When following redirects, the Client will forward all headers set on the
42 // initial [Request] except:
43 //
44 // - when forwarding sensitive headers like "Authorization",
45 // "WWW-Authenticate", and "Cookie" to untrusted targets.
46 // These headers will be ignored when following a redirect to a domain
47 // that is not a subdomain match or exact match of the initial domain.
48 // For example, a redirect from "foo.com" to either "foo.com" or "sub.foo.com"
49 // will forward the sensitive headers, but a redirect to "bar.com" will not.
50 // - when forwarding the "Cookie" header with a non-nil cookie Jar.
51 // Since each redirect may mutate the state of the cookie jar,
52 // a redirect may possibly alter a cookie set in the initial request.
53 // When forwarding the "Cookie" header, any mutated cookies will be omitted,
54 // with the expectation that the Jar will insert those mutated cookies
55 // with the updated values (assuming the origin matches).
56 // If Jar is nil, the initial cookies are forwarded without change.
57 type Client struct {
58 // Transport specifies the mechanism by which individual
59 // HTTP requests are made.
60 // If nil, DefaultTransport is used.
61 Transport RoundTripper
62 63 // CheckRedirect specifies the policy for handling redirects.
64 // If CheckRedirect is not nil, the client calls it before
65 // following an HTTP redirect. The arguments req and via are
66 // the upcoming request and the requests made already, oldest
67 // first. If CheckRedirect returns an error, the Client's Get
68 // method returns both the previous Response (with its Body
69 // closed) and CheckRedirect's error (wrapped in a url.Error)
70 // instead of issuing the Request req.
71 // As a special case, if CheckRedirect returns ErrUseLastResponse,
72 // then the most recent response is returned with its body
73 // unclosed, along with a nil error.
74 //
75 // If CheckRedirect is nil, the Client uses its default policy,
76 // which is to stop after 10 consecutive requests.
77 CheckRedirect func(req *Request, via []*Request) error
78 79 // Jar specifies the cookie jar.
80 //
81 // The Jar is used to insert relevant cookies into every
82 // outbound Request and is updated with the cookie values
83 // of every inbound Response. The Jar is consulted for every
84 // redirect that the Client follows.
85 //
86 // If Jar is nil, cookies are only sent if they are explicitly
87 // set on the Request.
88 Jar CookieJar
89 90 // Timeout specifies a time limit for requests made by this
91 // Client. The timeout includes connection time, any
92 // redirects, and reading the response body. The timer remains
93 // running after Get, Head, Post, or Do return and will
94 // interrupt reading of the Response.Body.
95 //
96 // A Timeout of zero means no timeout.
97 //
98 // The Client cancels requests to the underlying Transport
99 // as if the Request's Context ended.
100 //
101 // For compatibility, the Client will also use the deprecated
102 // CancelRequest method on Transport if found. New
103 // RoundTripper implementations should use the Request's Context
104 // for cancellation instead of implementing CancelRequest.
105 Timeout time.Duration
106 }
107 108 // DefaultClient is the default [Client] and is used by [Get], [Head], and [Post].
109 var DefaultClient = &Client{}
110 111 // RoundTripper is an interface representing the ability to execute a
112 // single HTTP transaction, obtaining the [Response] for a given [Request].
113 //
114 // A RoundTripper must be safe for concurrent use by multiple
115 // goroutines.
116 type RoundTripper interface {
117 // RoundTrip executes a single HTTP transaction, returning
118 // a Response for the provided Request.
119 //
120 // RoundTrip should not attempt to interpret the response. In
121 // particular, RoundTrip must return err == nil if it obtained
122 // a response, regardless of the response's HTTP status code.
123 // A non-nil err should be reserved for failure to obtain a
124 // response. Similarly, RoundTrip should not attempt to
125 // handle higher-level protocol details such as redirects,
126 // authentication, or cookies.
127 //
128 // RoundTrip should not modify the request, except for
129 // consuming and closing the Request's Body. RoundTrip may
130 // read fields of the request in a separate goroutine. Callers
131 // should not mutate or reuse the request until the Response's
132 // Body has been closed.
133 //
134 // RoundTrip must always close the body, including on errors,
135 // but depending on the implementation may do so in a separate
136 // goroutine even after RoundTrip returns. This means that
137 // callers wanting to reuse the body for subsequent requests
138 // must arrange to wait for the Close call before doing so.
139 //
140 // The Request's URL and Header fields must be initialized.
141 RoundTrip(*Request) (*Response, error)
142 }
143 144 // refererForURL returns a referer without any authentication info or
145 // an empty string if lastReq scheme is https and newReq scheme is http.
146 // If the referer was explicitly set, then it will continue to be used.
147 func refererForURL(lastReq, newReq *url.URL, explicitRef string) string {
148 // https://tools.ietf.org/html/rfc7231#section-5.5.2
149 // "Clients SHOULD NOT include a Referer header field in a
150 // (non-secure) HTTP request if the referring page was
151 // transferred with a secure protocol."
152 if lastReq.Scheme == "https" && newReq.Scheme == "http" {
153 return ""
154 }
155 if explicitRef != "" {
156 return explicitRef
157 }
158 159 referer := lastReq.String()
160 if lastReq.User != nil {
161 // This is not very efficient, but is the best we can
162 // do without:
163 // - introducing a new method on URL
164 // - creating a race condition
165 // - copying the URL struct manually, which would cause
166 // maintenance problems down the line
167 auth := lastReq.User.String() + "@"
168 referer = bytes.Replace(referer, auth, "", 1)
169 }
170 return referer
171 }
172 173 // didTimeout is non-nil only if err != nil.
174 func (c *Client) send(req *Request, deadline time.Time) (resp *Response, didTimeout func() bool, err error) {
175 if c.Jar != nil {
176 for _, cookie := range c.Jar.Cookies(req.URL) {
177 req.AddCookie(cookie)
178 }
179 }
180 resp, didTimeout, err = send(req, c.transport(), deadline)
181 if err != nil {
182 return nil, didTimeout, err
183 }
184 if c.Jar != nil {
185 if rc := resp.Cookies(); len(rc) > 0 {
186 c.Jar.SetCookies(req.URL, rc)
187 }
188 }
189 return resp, nil, nil
190 }
191 192 func (c *Client) deadline() time.Time {
193 if c.Timeout > 0 {
194 return time.Now().Add(c.Timeout)
195 }
196 return time.Time{}
197 }
198 199 func (c *Client) transport() RoundTripper {
200 if c.Transport != nil {
201 return c.Transport
202 }
203 return DefaultTransport
204 }
205 206 // ErrSchemeMismatch is returned when a server returns an HTTP response to an HTTPS client.
207 var ErrSchemeMismatch = errors.New("http: server gave HTTP response to HTTPS client")
208 209 // send issues an HTTP request.
210 // Caller should close resp.Body when done reading from it.
211 func send(ireq *Request, rt RoundTripper, deadline time.Time) (resp *Response, didTimeout func() bool, err error) {
212 req := ireq // req is either the original request, or a modified fork
213 214 if rt == nil {
215 req.closeBody()
216 return nil, alwaysFalse, errors.New("http: no Client.Transport or DefaultTransport")
217 }
218 219 if req.URL == nil {
220 req.closeBody()
221 return nil, alwaysFalse, errors.New("http: nil Request.URL")
222 }
223 224 if req.RequestURI != "" {
225 req.closeBody()
226 return nil, alwaysFalse, errors.New("http: Request.RequestURI can't be set in client requests")
227 }
228 229 // forkReq forks req into a shallow clone of ireq the first
230 // time it's called.
231 forkReq := func() {
232 if ireq == req {
233 req = &Request{}
234 *req = *ireq // shallow clone
235 }
236 }
237 238 // Most the callers of send (Get, Post, et al) don't need
239 // Headers, leaving it uninitialized. We guarantee to the
240 // Transport that this has been initialized, though.
241 if req.Header == nil {
242 forkReq()
243 req.Header = make(Header)
244 }
245 246 if u := req.URL.User; u != nil && req.Header.Get("Authorization") == "" {
247 username := u.Username()
248 password, _ := u.Password()
249 forkReq()
250 req.Header = cloneOrMakeHeader(ireq.Header)
251 req.Header.Set("Authorization", "Basic "+basicAuth(username, password))
252 }
253 254 if !deadline.IsZero() {
255 forkReq()
256 }
257 stopTimer, didTimeout := setRequestCancel(req, rt, deadline)
258 259 resp, err = rt.RoundTrip(req)
260 if err != nil {
261 stopTimer()
262 if resp != nil {
263 log.Printf("RoundTripper returned a response & error; ignoring response")
264 }
265 if tlsErr, ok := err.(tls.RecordHeaderError); ok {
266 // If we get a bad TLS record header, check to see if the
267 // response looks like HTTP and give a more helpful error.
268 // See golang.org/issue/11111.
269 if string(tlsErr.RecordHeader[:]) == "HTTP/" {
270 err = ErrSchemeMismatch
271 }
272 }
273 return nil, didTimeout, err
274 }
275 if resp == nil {
276 return nil, didTimeout, fmt.Errorf("http: RoundTripper implementation (%T) returned a nil *Response with a nil error", rt)
277 }
278 if resp.Body == nil {
279 // The documentation on the Body field says “The http Client and Transport
280 // guarantee that Body is always non-nil, even on responses without a body
281 // or responses with a zero-length body.” Unfortunately, we didn't document
282 // that same constraint for arbitrary RoundTripper implementations, and
283 // RoundTripper implementations in the wild (mostly in tests) assume that
284 // they can use a nil Body to mean an empty one (similar to Request.Body).
285 // (See https://golang.org/issue/38095.)
286 //
287 // If the ContentLength allows the Body to be empty, fill in an empty one
288 // here to ensure that it is non-nil.
289 if resp.ContentLength > 0 && req.Method != "HEAD" {
290 return nil, didTimeout, fmt.Errorf("http: RoundTripper implementation (%T) returned a *Response with content length %d but a nil Body", rt, resp.ContentLength)
291 }
292 resp.Body = io.NopCloser(bytes.NewReader(""))
293 }
294 if !deadline.IsZero() {
295 resp.Body = &cancelTimerBody{
296 stop: stopTimer,
297 rc: resp.Body,
298 reqDidTimeout: didTimeout,
299 }
300 }
301 return resp, nil, nil
302 }
303 304 // timeBeforeContextDeadline reports whether the non-zero Time t is
305 // before ctx's deadline, if any. If ctx does not have a deadline, it
306 // always reports true (the deadline is considered infinite).
307 func timeBeforeContextDeadline(t time.Time, ctx context.Context) bool {
308 d, ok := ctx.Deadline()
309 if !ok {
310 return true
311 }
312 return t.Before(d)
313 }
314 315 // knownRoundTripperImpl reports whether rt is a RoundTripper that's
316 // maintained by the Go team and known to implement the latest
317 // optional semantics (notably contexts). The Request is used
318 // to check whether this particular request is using an alternate protocol,
319 // in which case we need to check the RoundTripper for that protocol.
320 func knownRoundTripperImpl(rt RoundTripper, req *Request) bool {
321 switch t := rt.(type) {
322 case *Transport:
323 if altRT := t.alternateRoundTripper(req); altRT != nil {
324 return knownRoundTripperImpl(altRT, req)
325 }
326 return true
327 case *http2Transport, http2noDialH2RoundTripper:
328 return true
329 }
330 // Moxie: removed reflect-based heuristic for external http2.Transport.
331 // The bundled http2 types are handled in the type switch above.
332 return false
333 }
334 335 // setRequestCancel sets req.Cancel and adds a deadline context to req
336 // if deadline is non-zero. The RoundTripper's type is used to
337 // determine whether the legacy CancelRequest behavior should be used.
338 //
339 // As background, there are three ways to cancel a request:
340 // First was Transport.CancelRequest. (deprecated)
341 // Second was Request.Cancel.
342 // Third was Request.Context.
343 // This function populates the second and third, and uses the first if it really needs to.
344 func setRequestCancel(req *Request, rt RoundTripper, deadline time.Time) (stopTimer func(), didTimeout func() bool) {
345 if deadline.IsZero() {
346 return nop, alwaysFalse
347 }
348 knownTransport := knownRoundTripperImpl(rt, req)
349 oldCtx := req.Context()
350 351 if req.Cancel == nil && knownTransport {
352 // If they already had a Request.Context that's
353 // expiring sooner, do nothing:
354 if !timeBeforeContextDeadline(deadline, oldCtx) {
355 return nop, alwaysFalse
356 }
357 358 var cancelCtx func()
359 req.ctx, cancelCtx = context.WithDeadline(oldCtx, deadline)
360 return cancelCtx, func() bool { return time.Now().After(deadline) }
361 }
362 initialReqCancel := req.Cancel // the user's original Request.Cancel, if any
363 364 var cancelCtx func()
365 if timeBeforeContextDeadline(deadline, oldCtx) {
366 req.ctx, cancelCtx = context.WithDeadline(oldCtx, deadline)
367 }
368 369 cancel := chan struct{}{}
370 req.Cancel = cancel
371 372 doCancel := func() {
373 // The second way in the func comment above:
374 close(cancel)
375 // The first way, used only for RoundTripper
376 // implementations written before Go 1.5 or Go 1.6.
377 type canceler interface{ CancelRequest(*Request) }
378 if v, ok := rt.(canceler); ok {
379 v.CancelRequest(req)
380 }
381 }
382 383 stopTimerCh := chan struct{}{}
384 stopTimer = sync.OnceFunc(func() {
385 close(stopTimerCh)
386 if cancelCtx != nil {
387 cancelCtx()
388 }
389 })
390 391 timer := time.NewTimer(time.Until(deadline))
392 var timedOut atomic.Bool
393 394 func() {
395 select {
396 case <-initialReqCancel:
397 doCancel()
398 timer.Stop()
399 case <-timer.C:
400 timedOut.Store(true)
401 doCancel()
402 case <-stopTimerCh:
403 timer.Stop()
404 }
405 }()
406 407 return stopTimer, timedOut.Load
408 }
409 410 // See 2 (end of page 4) https://www.ietf.org/rfc/rfc2617.txt
411 // "To receive authorization, the client sends the userid and password,
412 // separated by a single colon (":") character, within a base64
413 // encoded string in the credentials."
414 // It is not meant to be urlencoded.
415 func basicAuth(username, password string) string {
416 auth := username + ":" + password
417 return base64.StdEncoding.EncodeToString([]byte(auth))
418 }
419 420 // Get issues a GET to the specified URL. If the response is one of
421 // the following redirect codes, Get follows the redirect, up to a
422 // maximum of 10 redirects:
423 //
424 // 301 (Moved Permanently)
425 // 302 (Found)
426 // 303 (See Other)
427 // 307 (Temporary Redirect)
428 // 308 (Permanent Redirect)
429 //
430 // An error is returned if there were too many redirects or if there
431 // was an HTTP protocol error. A non-2xx response doesn't cause an
432 // error. Any returned error will be of type [*url.Error]. The url.Error
433 // value's Timeout method will report true if the request timed out.
434 //
435 // When err is nil, resp always contains a non-nil resp.Body.
436 // Caller should close resp.Body when done reading from it.
437 //
438 // Get is a wrapper around DefaultClient.Get.
439 //
440 // To make a request with custom headers, use [NewRequest] and
441 // DefaultClient.Do.
442 //
443 // To make a request with a specified context.Context, use [NewRequestWithContext]
444 // and DefaultClient.Do.
445 func Get(url string) (resp *Response, err error) {
446 return DefaultClient.Get(url)
447 }
448 449 // Get issues a GET to the specified URL. If the response is one of the
450 // following redirect codes, Get follows the redirect after calling the
451 // [Client.CheckRedirect] function:
452 //
453 // 301 (Moved Permanently)
454 // 302 (Found)
455 // 303 (See Other)
456 // 307 (Temporary Redirect)
457 // 308 (Permanent Redirect)
458 //
459 // An error is returned if the [Client.CheckRedirect] function fails
460 // or if there was an HTTP protocol error. A non-2xx response doesn't
461 // cause an error. Any returned error will be of type [*url.Error]. The
462 // url.Error value's Timeout method will report true if the request
463 // timed out.
464 //
465 // When err is nil, resp always contains a non-nil resp.Body.
466 // Caller should close resp.Body when done reading from it.
467 //
468 // To make a request with custom headers, use [NewRequest] and [Client.Do].
469 //
470 // To make a request with a specified context.Context, use [NewRequestWithContext]
471 // and Client.Do.
472 func (c *Client) Get(url string) (resp *Response, err error) {
473 req, err := NewRequest("GET", url, nil)
474 if err != nil {
475 return nil, err
476 }
477 return c.Do(req)
478 }
479 480 func alwaysFalse() bool { return false }
481 482 // ErrUseLastResponse can be returned by Client.CheckRedirect hooks to
483 // control how redirects are processed. If returned, the next request
484 // is not sent and the most recent response is returned with its body
485 // unclosed.
486 var ErrUseLastResponse = errors.New("net/http: use last response")
487 488 // checkRedirect calls either the user's configured CheckRedirect
489 // function, or the default.
490 func (c *Client) checkRedirect(req *Request, via []*Request) error {
491 fn := c.CheckRedirect
492 if fn == nil {
493 fn = defaultCheckRedirect
494 }
495 return fn(req, via)
496 }
497 498 // redirectBehavior describes what should happen when the
499 // client encounters a 3xx status code from the server.
500 func redirectBehavior(reqMethod string, resp *Response, ireq *Request) (redirectMethod string, shouldRedirect, includeBody bool) {
501 switch resp.StatusCode {
502 case 301, 302, 303:
503 redirectMethod = reqMethod
504 shouldRedirect = true
505 includeBody = false
506 507 // RFC 2616 allowed automatic redirection only with GET and
508 // HEAD requests. RFC 7231 lifts this restriction, but we still
509 // restrict other methods to GET to maintain compatibility.
510 // See Issue 18570.
511 if reqMethod != "GET" && reqMethod != "HEAD" {
512 redirectMethod = "GET"
513 }
514 case 307, 308:
515 redirectMethod = reqMethod
516 shouldRedirect = true
517 includeBody = true
518 519 if ireq.GetBody == nil && ireq.outgoingLength() != 0 {
520 // We had a request body, and 307/308 require
521 // re-sending it, but GetBody is not defined. So just
522 // return this response to the user instead of an
523 // error, like we did in Go 1.7 and earlier.
524 shouldRedirect = false
525 }
526 }
527 return redirectMethod, shouldRedirect, includeBody
528 }
529 530 // urlErrorOp returns the (*url.Error).Op value to use for the
531 // provided (*Request).Method value.
532 func urlErrorOp(method string) string {
533 if method == "" {
534 return "Get"
535 }
536 if lowerMethod, ok := ascii.ToLower(method); ok {
537 return method[:1] + lowerMethod[1:]
538 }
539 return method
540 }
541 542 // Do sends an HTTP request and returns an HTTP response, following
543 // policy (such as redirects, cookies, auth) as configured on the
544 // client.
545 //
546 // An error is returned if caused by client policy (such as
547 // CheckRedirect), or failure to speak HTTP (such as a network
548 // connectivity problem). A non-2xx status code doesn't cause an
549 // error.
550 //
551 // If the returned error is nil, the [Response] will contain a non-nil
552 // Body which the user is expected to close. If the Body is not both
553 // read to EOF and closed, the [Client]'s underlying [RoundTripper]
554 // (typically [Transport]) may not be able to re-use a persistent TCP
555 // connection to the server for a subsequent "keep-alive" request.
556 //
557 // The request Body, if non-nil, will be closed by the underlying
558 // Transport, even on errors. The Body may be closed asynchronously after
559 // Do returns.
560 //
561 // On error, any Response can be ignored. A non-nil Response with a
562 // non-nil error only occurs when CheckRedirect fails, and even then
563 // the returned [Response.Body] is already closed.
564 //
565 // Generally [Get], [Post], or [PostForm] will be used instead of Do.
566 //
567 // If the server replies with a redirect, the Client first uses the
568 // CheckRedirect function to determine whether the redirect should be
569 // followed. If permitted, a 301, 302, or 303 redirect causes
570 // subsequent requests to use HTTP method GET
571 // (or HEAD if the original request was HEAD), with no body.
572 // A 307 or 308 redirect preserves the original HTTP method and body,
573 // provided that the [Request.GetBody] function is defined.
574 // The [NewRequest] function automatically sets GetBody for common
575 // standard library body types.
576 //
577 // Any returned error will be of type [*url.Error]. The url.Error
578 // value's Timeout method will report true if the request timed out.
579 func (c *Client) Do(req *Request) (*Response, error) {
580 return c.do(req)
581 }
582 583 var testHookClientDoResult func(retres *Response, reterr error)
584 585 func (c *Client) do(req *Request) (retres *Response, reterr error) {
586 if testHookClientDoResult != nil {
587 defer func() { testHookClientDoResult(retres, reterr) }()
588 }
589 if req.URL == nil {
590 req.closeBody()
591 return nil, &url.Error{
592 Op: urlErrorOp(req.Method),
593 Err: errors.New("http: nil Request.URL"),
594 }
595 }
596 _ = *c // panic early if c is nil; see go.dev/issue/53521
597 598 var (
599 deadline = c.deadline()
600 reqs []*Request
601 resp *Response
602 copyHeaders = c.makeHeadersCopier(req)
603 reqBodyClosed = false // have we closed the current req.Body?
604 605 // Redirect behavior:
606 redirectMethod string
607 includeBody = true
608 stripSensitiveHeaders = false
609 )
610 uerr := func(err error) error {
611 // the body may have been closed already by c.send()
612 if !reqBodyClosed {
613 req.closeBody()
614 }
615 var urlStr string
616 if resp != nil && resp.Request != nil {
617 urlStr = stripPassword(resp.Request.URL)
618 } else {
619 urlStr = stripPassword(req.URL)
620 }
621 return &url.Error{
622 Op: urlErrorOp(reqs[0].Method),
623 URL: urlStr,
624 Err: err,
625 }
626 }
627 for {
628 // For all but the first request, create the next
629 // request hop and replace req.
630 if len(reqs) > 0 {
631 loc := resp.Header.Get("Location")
632 if loc == "" {
633 // While most 3xx responses include a Location, it is not
634 // required and 3xx responses without a Location have been
635 // observed in the wild. See issues #17773 and #49281.
636 return resp, nil
637 }
638 u, err := req.URL.Parse(loc)
639 if err != nil {
640 resp.closeBody()
641 return nil, uerr(fmt.Errorf("failed to parse Location header %q: %v", loc, err))
642 }
643 host := ""
644 if req.Host != "" && req.Host != req.URL.Host {
645 // If the caller specified a custom Host header and the
646 // redirect location is relative, preserve the Host header
647 // through the redirect. See issue #22233.
648 if u, _ := url.Parse(loc); u != nil && !u.IsAbs() {
649 host = req.Host
650 }
651 }
652 ireq := reqs[0]
653 req = &Request{
654 Method: redirectMethod,
655 Response: resp,
656 URL: u,
657 Header: make(Header),
658 Host: host,
659 Cancel: ireq.Cancel,
660 ctx: ireq.ctx,
661 }
662 if includeBody && ireq.GetBody != nil {
663 req.Body, err = ireq.GetBody()
664 if err != nil {
665 resp.closeBody()
666 return nil, uerr(err)
667 }
668 req.GetBody = ireq.GetBody
669 req.ContentLength = ireq.ContentLength
670 }
671 672 // Copy original headers before setting the Referer,
673 // in case the user set Referer on their first request.
674 // If they really want to override, they can do it in
675 // their CheckRedirect func.
676 if !stripSensitiveHeaders && reqs[0].URL.Host != req.URL.Host {
677 if !shouldCopyHeaderOnRedirect(reqs[0].URL, req.URL) {
678 stripSensitiveHeaders = true
679 }
680 }
681 copyHeaders(req, stripSensitiveHeaders)
682 683 // Add the Referer header from the most recent
684 // request URL to the new one, if it's not https->http:
685 if ref := refererForURL(reqs[len(reqs)-1].URL, req.URL, req.Header.Get("Referer")); ref != "" {
686 req.Header.Set("Referer", ref)
687 }
688 err = c.checkRedirect(req, reqs)
689 690 // Sentinel error to let users select the
691 // previous response, without closing its
692 // body. See Issue 10069.
693 if err == ErrUseLastResponse {
694 return resp, nil
695 }
696 697 // Close the previous response's body. But
698 // read at least some of the body so if it's
699 // small the underlying TCP connection will be
700 // re-used. No need to check for errors: if it
701 // fails, the Transport won't reuse it anyway.
702 const maxBodySlurpSize = 2 << 10
703 if resp.ContentLength == -1 || resp.ContentLength <= maxBodySlurpSize {
704 io.CopyN(io.Discard, resp.Body, maxBodySlurpSize)
705 }
706 resp.Body.Close()
707 708 if err != nil {
709 // Special case for Go 1 compatibility: return both the response
710 // and an error if the CheckRedirect function failed.
711 // See https://golang.org/issue/3795
712 // The resp.Body has already been closed.
713 ue := uerr(err)
714 ue.(*url.Error).URL = loc
715 return resp, ue
716 }
717 }
718 719 reqs = append(reqs, req)
720 var err error
721 var didTimeout func() bool
722 if resp, didTimeout, err = c.send(req, deadline); err != nil {
723 // c.send() always closes req.Body
724 reqBodyClosed = true
725 if !deadline.IsZero() && didTimeout() {
726 err = &timeoutError{err.Error() + " (Client.Timeout exceeded while awaiting headers)"}
727 }
728 return nil, uerr(err)
729 }
730 731 var shouldRedirect, includeBodyOnHop bool
732 redirectMethod, shouldRedirect, includeBodyOnHop = redirectBehavior(req.Method, resp, reqs[0])
733 if !shouldRedirect {
734 return resp, nil
735 }
736 if !includeBodyOnHop {
737 // Once a hop drops the body, we never send it again
738 // (because we're now handling a redirect for a request with no body).
739 includeBody = false
740 }
741 742 req.closeBody()
743 }
744 }
745 746 // makeHeadersCopier makes a function that copies headers from the
747 // initial Request, ireq. For every redirect, this function must be called
748 // so that it can copy headers into the upcoming Request.
749 func (c *Client) makeHeadersCopier(ireq *Request) func(req *Request, stripSensitiveHeaders bool) {
750 // The headers to copy are from the very initial request.
751 // We use a closured callback to keep a reference to these original headers.
752 var (
753 ireqhdr = cloneOrMakeHeader(ireq.Header)
754 icookies map[string][]*Cookie
755 )
756 if c.Jar != nil && ireq.Header.Get("Cookie") != "" {
757 icookies = map[string][]*Cookie{}
758 for _, c := range ireq.Cookies() {
759 icookies[c.Name] = append(icookies[c.Name], c)
760 }
761 }
762 763 return func(req *Request, stripSensitiveHeaders bool) {
764 // If Jar is present and there was some initial cookies provided
765 // via the request header, then we may need to alter the initial
766 // cookies as we follow redirects since each redirect may end up
767 // modifying a pre-existing cookie.
768 //
769 // Since cookies already set in the request header do not contain
770 // information about the original domain and path, the logic below
771 // assumes any new set cookies override the original cookie
772 // regardless of domain or path.
773 //
774 // See https://golang.org/issue/17494
775 if c.Jar != nil && icookies != nil {
776 var changed bool
777 resp := req.Response // The response that caused the upcoming redirect
778 for _, c := range resp.Cookies() {
779 if _, ok := icookies[c.Name]; ok {
780 delete(icookies, c.Name)
781 changed = true
782 }
783 }
784 if changed {
785 ireqhdr.Del("Cookie")
786 var ss [][]byte
787 for _, cs := range icookies {
788 for _, c := range cs {
789 ss = append(ss, c.Name+"="+c.Value)
790 }
791 }
792 slices.Sort(ss) // Ensure deterministic headers
793 ireqhdr.Set("Cookie", bytes.Join(ss, "; "))
794 }
795 }
796 797 // Copy the initial request's Header values
798 // (at least the safe ones).
799 for k, vv := range ireqhdr {
800 sensitive := false
801 switch CanonicalHeaderKey(k) {
802 case "Authorization", "Www-Authenticate", "Cookie", "Cookie2",
803 "Proxy-Authorization", "Proxy-Authenticate":
804 sensitive = true
805 }
806 if !(sensitive && stripSensitiveHeaders) {
807 req.Header[k] = vv
808 }
809 }
810 }
811 }
812 813 func defaultCheckRedirect(req *Request, via []*Request) error {
814 if len(via) >= 10 {
815 return errors.New("stopped after 10 redirects")
816 }
817 return nil
818 }
819 820 // Post issues a POST to the specified URL.
821 //
822 // Caller should close resp.Body when done reading from it.
823 //
824 // If the provided body is an [io.Closer], it is closed after the
825 // request.
826 //
827 // Post is a wrapper around DefaultClient.Post.
828 //
829 // To set custom headers, use [NewRequest] and DefaultClient.Do.
830 //
831 // See the [Client.Do] method documentation for details on how redirects
832 // are handled.
833 //
834 // To make a request with a specified context.Context, use [NewRequestWithContext]
835 // and DefaultClient.Do.
836 func Post(url, contentType string, body io.Reader) (resp *Response, err error) {
837 return DefaultClient.Post(url, contentType, body)
838 }
839 840 // Post issues a POST to the specified URL.
841 //
842 // Caller should close resp.Body when done reading from it.
843 //
844 // If the provided body is an [io.Closer], it is closed after the
845 // request.
846 //
847 // To set custom headers, use [NewRequest] and [Client.Do].
848 //
849 // To make a request with a specified context.Context, use [NewRequestWithContext]
850 // and [Client.Do].
851 //
852 // See the [Client.Do] method documentation for details on how redirects
853 // are handled.
854 func (c *Client) Post(url, contentType string, body io.Reader) (resp *Response, err error) {
855 req, err := NewRequest("POST", url, body)
856 if err != nil {
857 return nil, err
858 }
859 req.Header.Set("Content-Type", contentType)
860 return c.Do(req)
861 }
862 863 // PostForm issues a POST to the specified URL, with data's keys and
864 // values URL-encoded as the request body.
865 //
866 // The Content-Type header is set to application/x-www-form-urlencoded.
867 // To set other headers, use [NewRequest] and DefaultClient.Do.
868 //
869 // When err is nil, resp always contains a non-nil resp.Body.
870 // Caller should close resp.Body when done reading from it.
871 //
872 // PostForm is a wrapper around DefaultClient.PostForm.
873 //
874 // See the [Client.Do] method documentation for details on how redirects
875 // are handled.
876 //
877 // To make a request with a specified [context.Context], use [NewRequestWithContext]
878 // and DefaultClient.Do.
879 func PostForm(url string, data url.Values) (resp *Response, err error) {
880 return DefaultClient.PostForm(url, data)
881 }
882 883 // PostForm issues a POST to the specified URL,
884 // with data's keys and values URL-encoded as the request body.
885 //
886 // The Content-Type header is set to application/x-www-form-urlencoded.
887 // To set other headers, use [NewRequest] and [Client.Do].
888 //
889 // When err is nil, resp always contains a non-nil resp.Body.
890 // Caller should close resp.Body when done reading from it.
891 //
892 // See the [Client.Do] method documentation for details on how redirects
893 // are handled.
894 //
895 // To make a request with a specified context.Context, use [NewRequestWithContext]
896 // and Client.Do.
897 func (c *Client) PostForm(url string, data url.Values) (resp *Response, err error) {
898 return c.Post(url, "application/x-www-form-urlencoded", bytes.NewReader(data.Encode()))
899 }
900 901 // Head issues a HEAD to the specified URL. If the response is one of
902 // the following redirect codes, Head follows the redirect, up to a
903 // maximum of 10 redirects:
904 //
905 // 301 (Moved Permanently)
906 // 302 (Found)
907 // 303 (See Other)
908 // 307 (Temporary Redirect)
909 // 308 (Permanent Redirect)
910 //
911 // Head is a wrapper around DefaultClient.Head.
912 //
913 // To make a request with a specified [context.Context], use [NewRequestWithContext]
914 // and DefaultClient.Do.
915 func Head(url string) (resp *Response, err error) {
916 return DefaultClient.Head(url)
917 }
918 919 // Head issues a HEAD to the specified URL. If the response is one of the
920 // following redirect codes, Head follows the redirect after calling the
921 // [Client.CheckRedirect] function:
922 //
923 // 301 (Moved Permanently)
924 // 302 (Found)
925 // 303 (See Other)
926 // 307 (Temporary Redirect)
927 // 308 (Permanent Redirect)
928 //
929 // To make a request with a specified [context.Context], use [NewRequestWithContext]
930 // and [Client.Do].
931 func (c *Client) Head(url string) (resp *Response, err error) {
932 req, err := NewRequest("HEAD", url, nil)
933 if err != nil {
934 return nil, err
935 }
936 return c.Do(req)
937 }
938 939 // CloseIdleConnections closes any connections on its [Transport] which
940 // were previously connected from previous requests but are now
941 // sitting idle in a "keep-alive" state. It does not interrupt any
942 // connections currently in use.
943 //
944 // If [Client.Transport] does not have a [Client.CloseIdleConnections] method
945 // then this method does nothing.
946 func (c *Client) CloseIdleConnections() {
947 type closeIdler interface {
948 CloseIdleConnections()
949 }
950 if tr, ok := c.transport().(closeIdler); ok {
951 tr.CloseIdleConnections()
952 }
953 }
954 955 // cancelTimerBody is an io.ReadCloser that wraps rc with two features:
956 // 1. On Read error or close, the stop func is called.
957 // 2. On Read failure, if reqDidTimeout is true, the error is wrapped and
958 // marked as net.Error that hit its timeout.
959 type cancelTimerBody struct {
960 stop func() // stops the time.Timer waiting to cancel the request
961 rc io.ReadCloser
962 reqDidTimeout func() bool
963 }
964 965 func (b *cancelTimerBody) Read(p []byte) (n int, err error) {
966 n, err = b.rc.Read(p)
967 if err == nil {
968 return n, nil
969 }
970 if err == io.EOF {
971 return n, err
972 }
973 if b.reqDidTimeout() {
974 err = &timeoutError{err.Error() + " (Client.Timeout or context cancellation while reading body)"}
975 }
976 return n, err
977 }
978 979 func (b *cancelTimerBody) Close() error {
980 err := b.rc.Close()
981 b.stop()
982 return err
983 }
984 985 func shouldCopyHeaderOnRedirect(initial, dest *url.URL) bool {
986 // Permit sending auth/cookie headers from "foo.com"
987 // to "sub.foo.com".
988 989 // Note that we don't send all cookies to subdomains
990 // automatically. This function is only used for
991 // Cookies set explicitly on the initial outgoing
992 // client request. Cookies automatically added via the
993 // CookieJar mechanism continue to follow each
994 // cookie's scope as set by Set-Cookie. But for
995 // outgoing requests with the Cookie header set
996 // directly, we don't know their scope, so we assume
997 // it's for *.domain.com.
998 999 ihost := idnaASCIIFromURL(initial)
1000 dhost := idnaASCIIFromURL(dest)
1001 return isDomainOrSubdomain(dhost, ihost)
1002 }
1003 1004 // isDomainOrSubdomain reports whether sub is a subdomain (or exact
1005 // match) of the parent domain.
1006 //
1007 // Both domains must already be in canonical form.
1008 func isDomainOrSubdomain(sub, parent string) bool {
1009 if sub == parent {
1010 return true
1011 }
1012 // If sub contains a :, it's probably an IPv6 address (and is definitely not a hostname).
1013 // Don't check the suffix in this case, to avoid matching the contents of a IPv6 zone.
1014 // For example, "::1%.www.example.com" is not a subdomain of "www.example.com".
1015 if bytes.ContainsAny(sub, ":%") {
1016 return false
1017 }
1018 // If sub is "foo.example.com" and parent is "example.com",
1019 // that means sub must end in "."+parent.
1020 // Do it without allocating.
1021 if !bytes.HasSuffix(sub, parent) {
1022 return false
1023 }
1024 return sub[len(sub)-len(parent)-1] == '.'
1025 }
1026 1027 func stripPassword(u *url.URL) string {
1028 _, passSet := u.User.Password()
1029 if passSet {
1030 return bytes.Replace(u.String(), u.User.String()+"@", u.User.Username()+":***@", 1)
1031 }
1032 return u.String()
1033 }
1034