1 // Copyright 2011 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
4 5 // HTTP client implementation. See RFC 7230 through 7235.
6 //
7 // This is the low-level Transport implementation of RoundTripper.
8 // The high-level interface is in client.go.
9 10 package http
11 12 import (
13 "bufio"
14 "compress/gzip"
15 "container/list"
16 "context"
17 "crypto/tls"
18 "errors"
19 "fmt"
20 "internal/godebug"
21 "io"
22 "log"
23 "maps"
24 "net"
25 "net/http/httptrace"
26 "net/http/internal/ascii"
27 "net/textproto"
28 "net/url"
29 30 "bytes"
31 "sync"
32 "sync/atomic"
33 "time"
34 _ "unsafe"
35 36 "golang.org/x/net/http/httpguts"
37 "golang.org/x/net/http/httpproxy"
38 )
39 40 // DefaultTransport is the default implementation of [Transport] and is
41 // used by [DefaultClient]. It establishes network connections as needed
42 // and caches them for reuse by subsequent calls. It uses HTTP proxies
43 // as directed by the environment variables HTTP_PROXY, HTTPS_PROXY
44 // and NO_PROXY (or the lowercase versions thereof).
45 var DefaultTransport RoundTripper = &Transport{
46 Proxy: ProxyFromEnvironment,
47 DialContext: defaultTransportDialContext(&net.Dialer{
48 Timeout: 30 * time.Second,
49 KeepAlive: 30 * time.Second,
50 }),
51 ForceAttemptHTTP2: true,
52 MaxIdleConns: 100,
53 IdleConnTimeout: 90 * time.Second,
54 TLSHandshakeTimeout: 10 * time.Second,
55 ExpectContinueTimeout: 1 * time.Second,
56 }
57 58 // DefaultMaxIdleConnsPerHost is the default value of [Transport]'s
59 // MaxIdleConnsPerHost.
60 const DefaultMaxIdleConnsPerHost = 2
61 62 // Transport is an implementation of [RoundTripper] that supports HTTP,
63 // HTTPS, and HTTP proxies (for either HTTP or HTTPS with CONNECT).
64 //
65 // By default, Transport caches connections for future re-use.
66 // This may leave many open connections when accessing many hosts.
67 // This behavior can be managed using [Transport.CloseIdleConnections] method
68 // and the [Transport.MaxIdleConnsPerHost] and [Transport.DisableKeepAlives] fields.
69 //
70 // Transports should be reused instead of created as needed.
71 // Transports are safe for concurrent use by multiple goroutines.
72 //
73 // A Transport is a low-level primitive for making HTTP and HTTPS requests.
74 // For high-level functionality, such as cookies and redirects, see [Client].
75 //
76 // Transport uses HTTP/1.1 for HTTP URLs and either HTTP/1.1 or HTTP/2
77 // for HTTPS URLs, depending on whether the server supports HTTP/2,
78 // and how the Transport is configured. The [DefaultTransport] supports HTTP/2.
79 // To explicitly enable HTTP/2 on a transport, set [Transport.Protocols].
80 //
81 // Responses with status codes in the 1xx range are either handled
82 // automatically (100 expect-continue) or ignored. The one
83 // exception is HTTP status code 101 (Switching Protocols), which is
84 // considered a terminal status and returned by [Transport.RoundTrip]. To see the
85 // ignored 1xx responses, use the httptrace trace package's
86 // ClientTrace.Got1xxResponse.
87 //
88 // Transport only retries a request upon encountering a network error
89 // if the connection has been already been used successfully and if the
90 // request is idempotent and either has no body or has its [Request.GetBody]
91 // defined. HTTP requests are considered idempotent if they have HTTP methods
92 // GET, HEAD, OPTIONS, or TRACE; or if their [Header] map contains an
93 // "Idempotency-Key" or "X-Idempotency-Key" entry. If the idempotency key
94 // value is a zero-length slice, the request is treated as idempotent but the
95 // header is not sent on the wire.
96 type Transport struct {
97 idleMu sync.Mutex
98 closeIdle bool // user has requested to close all idle conns
99 idleConn map[connectMethodKey][]*persistConn // most recently used at end
100 idleConnWait map[connectMethodKey]wantConnQueue // waiting getConns
101 idleLRU connLRU
102 103 reqMu sync.Mutex
104 reqCanceler map[*Request]context.CancelCauseFunc
105 106 altMu sync.Mutex // guards changing altProto only
107 altProto atomic.Value // of nil or map[string]RoundTripper, key is URI scheme
108 109 connsPerHostMu sync.Mutex
110 connsPerHost map[connectMethodKey]int
111 connsPerHostWait map[connectMethodKey]wantConnQueue // waiting getConns
112 dialsInProgress wantConnQueue
113 114 // Proxy specifies a function to return a proxy for a given
115 // Request. If the function returns a non-nil error, the
116 // request is aborted with the provided error.
117 //
118 // The proxy type is determined by the URL scheme. "http",
119 // "https", "socks5", and "socks5h" are supported. If the scheme is empty,
120 // "http" is assumed.
121 // "socks5" is treated the same as "socks5h".
122 //
123 // If the proxy URL contains a userinfo subcomponent,
124 // the proxy request will pass the username and password
125 // in a Proxy-Authorization header.
126 //
127 // If Proxy is nil or returns a nil *URL, no proxy is used.
128 Proxy func(*Request) (*url.URL, error)
129 130 // OnProxyConnectResponse is called when the Transport gets an HTTP response from
131 // a proxy for a CONNECT request. It's called before the check for a 200 OK response.
132 // If it returns an error, the request fails with that error.
133 OnProxyConnectResponse func(ctx context.Context, proxyURL *url.URL, connectReq *Request, connectRes *Response) error
134 135 // DialContext specifies the dial function for creating unencrypted TCP connections.
136 // If DialContext is nil (and the deprecated Dial below is also nil),
137 // then the transport dials using package net.
138 //
139 // DialContext runs concurrently with calls to RoundTrip.
140 // A RoundTrip call that initiates a dial may end up using
141 // a connection dialed previously when the earlier connection
142 // becomes idle before the later DialContext completes.
143 DialContext func(ctx context.Context, network, addr []byte) (net.Conn, error)
144 145 // Dial specifies the dial function for creating unencrypted TCP connections.
146 //
147 // Dial runs concurrently with calls to RoundTrip.
148 // A RoundTrip call that initiates a dial may end up using
149 // a connection dialed previously when the earlier connection
150 // becomes idle before the later Dial completes.
151 //
152 // Deprecated: Use DialContext instead, which allows the transport
153 // to cancel dials as soon as they are no longer needed.
154 // If both are set, DialContext takes priority.
155 Dial func(network, addr string) (net.Conn, error)
156 157 // DialTLSContext specifies an optional dial function for creating
158 // TLS connections for non-proxied HTTPS requests.
159 //
160 // If DialTLSContext is nil (and the deprecated DialTLS below is also nil),
161 // DialContext and TLSClientConfig are used.
162 //
163 // If DialTLSContext is set, the Dial and DialContext hooks are not used for HTTPS
164 // requests and the TLSClientConfig and TLSHandshakeTimeout
165 // are ignored. The returned net.Conn is assumed to already be
166 // past the TLS handshake.
167 DialTLSContext func(ctx context.Context, network, addr string) (net.Conn, error)
168 169 // DialTLS specifies an optional dial function for creating
170 // TLS connections for non-proxied HTTPS requests.
171 //
172 // Deprecated: Use DialTLSContext instead, which allows the transport
173 // to cancel dials as soon as they are no longer needed.
174 // If both are set, DialTLSContext takes priority.
175 DialTLS func(network, addr string) (net.Conn, error)
176 177 // TLSClientConfig specifies the TLS configuration to use with
178 // tls.Client.
179 // If nil, the default configuration is used.
180 // If non-nil, HTTP/2 support may not be enabled by default.
181 TLSClientConfig *tls.Config
182 183 // TLSHandshakeTimeout specifies the maximum amount of time to
184 // wait for a TLS handshake. Zero means no timeout.
185 TLSHandshakeTimeout time.Duration
186 187 // DisableKeepAlives, if true, disables HTTP keep-alives and
188 // will only use the connection to the server for a single
189 // HTTP request.
190 //
191 // This is unrelated to the similarly named TCP keep-alives.
192 DisableKeepAlives bool
193 194 // DisableCompression, if true, prevents the Transport from
195 // requesting compression with an "Accept-Encoding: gzip"
196 // request header when the Request contains no existing
197 // Accept-Encoding value. If the Transport requests gzip on
198 // its own and gets a gzipped response, it's transparently
199 // decoded in the Response.Body. However, if the user
200 // explicitly requested gzip it is not automatically
201 // uncompressed.
202 DisableCompression bool
203 204 // MaxIdleConns controls the maximum number of idle (keep-alive)
205 // connections across all hosts. Zero means no limit.
206 MaxIdleConns int
207 208 // MaxIdleConnsPerHost, if non-zero, controls the maximum idle
209 // (keep-alive) connections to keep per-host. If zero,
210 // DefaultMaxIdleConnsPerHost is used.
211 MaxIdleConnsPerHost int
212 213 // MaxConnsPerHost optionally limits the total number of
214 // connections per host, including connections in the dialing,
215 // active, and idle states. On limit violation, dials will block.
216 //
217 // Zero means no limit.
218 MaxConnsPerHost int
219 220 // IdleConnTimeout is the maximum amount of time an idle
221 // (keep-alive) connection will remain idle before closing
222 // itself.
223 // Zero means no limit.
224 IdleConnTimeout time.Duration
225 226 // ResponseHeaderTimeout, if non-zero, specifies the amount of
227 // time to wait for a server's response headers after fully
228 // writing the request (including its body, if any). This
229 // time does not include the time to read the response body.
230 ResponseHeaderTimeout time.Duration
231 232 // ExpectContinueTimeout, if non-zero, specifies the amount of
233 // time to wait for a server's first response headers after fully
234 // writing the request headers if the request has an
235 // "Expect: 100-continue" header. Zero means no timeout and
236 // causes the body to be sent immediately, without
237 // waiting for the server to approve.
238 // This time does not include the time to send the request header.
239 ExpectContinueTimeout time.Duration
240 241 // TLSNextProto specifies how the Transport switches to an
242 // alternate protocol (such as HTTP/2) after a TLS ALPN
243 // protocol negotiation. If Transport dials a TLS connection
244 // with a non-empty protocol name and TLSNextProto contains a
245 // map entry for that key (such as "h2"), then the func is
246 // called with the request's authority (such as "example.com"
247 // or "example.com:1234") and the TLS connection. The function
248 // must return a RoundTripper that then handles the request.
249 // If TLSNextProto is not nil, HTTP/2 support is not enabled
250 // automatically.
251 TLSNextProto map[string]func(authority string, c *tls.Conn) RoundTripper
252 253 // ProxyConnectHeader optionally specifies headers to send to
254 // proxies during CONNECT requests.
255 // To set the header dynamically, see GetProxyConnectHeader.
256 ProxyConnectHeader Header
257 258 // GetProxyConnectHeader optionally specifies a func to return
259 // headers to send to proxyURL during a CONNECT request to the
260 // ip:port target.
261 // If it returns an error, the Transport's RoundTrip fails with
262 // that error. It can return (nil, nil) to not add headers.
263 // If GetProxyConnectHeader is non-nil, ProxyConnectHeader is
264 // ignored.
265 GetProxyConnectHeader func(ctx context.Context, proxyURL *url.URL, target string) (Header, error)
266 267 // MaxResponseHeaderBytes specifies a limit on how many
268 // response bytes are allowed in the server's response
269 // header.
270 //
271 // Zero means to use a default limit.
272 MaxResponseHeaderBytes int64
273 274 // WriteBufferSize specifies the size of the write buffer used
275 // when writing to the transport.
276 // If zero, a default (currently 4KB) is used.
277 WriteBufferSize int
278 279 // ReadBufferSize specifies the size of the read buffer used
280 // when reading from the transport.
281 // If zero, a default (currently 4KB) is used.
282 ReadBufferSize int
283 284 // nextProtoOnce guards initialization of TLSNextProto and
285 // h2transport (via onceSetNextProtoDefaults)
286 nextProtoOnce sync.Once
287 h2transport h2Transport // non-nil if http2 wired up
288 tlsNextProtoWasNil bool // whether TLSNextProto was nil when the Once fired
289 290 // ForceAttemptHTTP2 controls whether HTTP/2 is enabled when a non-zero
291 // Dial, DialTLS, or DialContext func or TLSClientConfig is provided.
292 // By default, use of any those fields conservatively disables HTTP/2.
293 // To use a custom dialer or TLS config and still attempt HTTP/2
294 // upgrades, set this to true.
295 ForceAttemptHTTP2 bool
296 297 // HTTP2 configures HTTP/2 connections.
298 //
299 // This field does not yet have any effect.
300 // See https://go.dev/issue/67813.
301 HTTP2 *HTTP2Config
302 303 // Protocols is the set of protocols supported by the transport.
304 //
305 // If Protocols includes UnencryptedHTTP2 and does not include HTTP1,
306 // the transport will use unencrypted HTTP/2 for requests for http:// URLs.
307 //
308 // If Protocols is nil, the default is usually HTTP/1 only.
309 // If ForceAttemptHTTP2 is true, or if TLSNextProto contains an "h2" entry,
310 // the default is HTTP/1 and HTTP/2.
311 Protocols *Protocols
312 }
313 314 func (t *Transport) writeBufferSize() int {
315 if t.WriteBufferSize > 0 {
316 return t.WriteBufferSize
317 }
318 return 4 << 10
319 }
320 321 func (t *Transport) readBufferSize() int {
322 if t.ReadBufferSize > 0 {
323 return t.ReadBufferSize
324 }
325 return 4 << 10
326 }
327 328 // Clone returns a deep copy of t's exported fields.
329 func (t *Transport) Clone() *Transport {
330 t.nextProtoOnce.Do(t.onceSetNextProtoDefaults)
331 t2 := &Transport{
332 Proxy: t.Proxy,
333 OnProxyConnectResponse: t.OnProxyConnectResponse,
334 DialContext: t.DialContext,
335 Dial: t.Dial,
336 DialTLS: t.DialTLS,
337 DialTLSContext: t.DialTLSContext,
338 TLSHandshakeTimeout: t.TLSHandshakeTimeout,
339 DisableKeepAlives: t.DisableKeepAlives,
340 DisableCompression: t.DisableCompression,
341 MaxIdleConns: t.MaxIdleConns,
342 MaxIdleConnsPerHost: t.MaxIdleConnsPerHost,
343 MaxConnsPerHost: t.MaxConnsPerHost,
344 IdleConnTimeout: t.IdleConnTimeout,
345 ResponseHeaderTimeout: t.ResponseHeaderTimeout,
346 ExpectContinueTimeout: t.ExpectContinueTimeout,
347 ProxyConnectHeader: t.ProxyConnectHeader.Clone(),
348 GetProxyConnectHeader: t.GetProxyConnectHeader,
349 MaxResponseHeaderBytes: t.MaxResponseHeaderBytes,
350 ForceAttemptHTTP2: t.ForceAttemptHTTP2,
351 WriteBufferSize: t.WriteBufferSize,
352 ReadBufferSize: t.ReadBufferSize,
353 }
354 if t.TLSClientConfig != nil {
355 t2.TLSClientConfig = t.TLSClientConfig.Clone()
356 }
357 if t.HTTP2 != nil {
358 t2.HTTP2 = &HTTP2Config{}
359 *t2.HTTP2 = *t.HTTP2
360 }
361 if t.Protocols != nil {
362 t2.Protocols = &Protocols{}
363 *t2.Protocols = *t.Protocols
364 }
365 if !t.tlsNextProtoWasNil {
366 npm := maps.Clone(t.TLSNextProto)
367 if npm == nil {
368 npm = map[string]func(authority string, c *tls.Conn) RoundTripper{}
369 }
370 t2.TLSNextProto = npm
371 }
372 return t2
373 }
374 375 // h2Transport is the interface we expect to be able to call from
376 // net/http against an *http2.Transport that's either bundled into
377 // h2_bundle.go or supplied by the user via x/net/http2.
378 //
379 // We name it with the "h2" prefix to stay out of the "http2" prefix
380 // namespace used by x/tools/cmd/bundle for h2_bundle.go.
381 type h2Transport interface {
382 CloseIdleConnections()
383 }
384 385 func (t *Transport) hasCustomTLSDialer() bool {
386 return t.DialTLS != nil || t.DialTLSContext != nil
387 }
388 389 var http2client = godebug.New("http2client")
390 391 // onceSetNextProtoDefaults initializes TLSNextProto.
392 // It must be called via t.nextProtoOnce.Do.
393 func (t *Transport) onceSetNextProtoDefaults() {
394 t.tlsNextProtoWasNil = (t.TLSNextProto == nil)
395 if http2client.Value() == "0" {
396 http2client.IncNonDefault()
397 return
398 }
399 400 // If they've already configured http2 with
401 // golang.org/x/net/http2 instead of the bundled copy, try to
402 // get at its http2.Transport value (via the "https"
403 // altproto map) so we can call CloseIdleConnections on it if
404 // requested. (Issue 22891)
405 altProto, _ := t.altProto.Load().(map[string]RoundTripper)
406 // Moxie: removed reflect-based extraction of external http2.Transport.
407 // Try direct type assertion instead.
408 if ap := altProto["https"]; ap != nil {
409 if h2i, ok := ap.(h2Transport); ok {
410 t.h2transport = h2i
411 return
412 }
413 }
414 415 if _, ok := t.TLSNextProto["h2"]; ok {
416 // There's an existing HTTP/2 implementation installed.
417 return
418 }
419 protocols := t.protocols()
420 if !protocols.HTTP2() && !protocols.UnencryptedHTTP2() {
421 return
422 }
423 if omitBundledHTTP2 {
424 return
425 }
426 t2, err := http2configureTransports(t)
427 if err != nil {
428 log.Printf("Error enabling Transport HTTP/2 support: %v", err)
429 return
430 }
431 t.h2transport = t2
432 433 // Auto-configure the http2.Transport's MaxHeaderListSize from
434 // the http.Transport's MaxResponseHeaderBytes. They don't
435 // exactly mean the same thing, but they're close.
436 //
437 // TODO: also add this to x/net/http2.Configure Transport, behind
438 // a +build go1.7 build tag:
439 if limit1 := t.MaxResponseHeaderBytes; limit1 != 0 && t2.MaxHeaderListSize == 0 {
440 const h2max = 1<<32 - 1
441 if limit1 >= h2max {
442 t2.MaxHeaderListSize = h2max
443 } else {
444 t2.MaxHeaderListSize = uint32(limit1)
445 }
446 }
447 448 // Server.ServeTLS clones the tls.Config before modifying it.
449 // Transport doesn't. We may want to make the two consistent some day.
450 //
451 // http2configureTransport will have already set NextProtos, but adjust it again
452 // here to remove HTTP/1.1 if the user has disabled it.
453 t.TLSClientConfig.NextProtos = adjustNextProtos(t.TLSClientConfig.NextProtos, protocols)
454 }
455 456 func (t *Transport) protocols() Protocols {
457 if t.Protocols != nil {
458 return *t.Protocols // user-configured set
459 }
460 var p Protocols
461 p.SetHTTP1(true) // default always includes HTTP/1
462 switch {
463 case t.TLSNextProto != nil:
464 // Setting TLSNextProto to an empty map is a documented way
465 // to disable HTTP/2 on a Transport.
466 if t.TLSNextProto["h2"] != nil {
467 p.SetHTTP2(true)
468 }
469 case !t.ForceAttemptHTTP2 && (t.TLSClientConfig != nil || t.Dial != nil || t.DialContext != nil || t.hasCustomTLSDialer()):
470 // Be conservative and don't automatically enable
471 // http2 if they've specified a custom TLS config or
472 // custom dialers. Let them opt-in themselves via
473 // Transport.Protocols.SetHTTP2(true) so we don't surprise them
474 // by modifying their tls.Config. Issue 14275.
475 // However, if ForceAttemptHTTP2 is true, it overrides the above checks.
476 case http2client.Value() == "0":
477 default:
478 p.SetHTTP2(true)
479 }
480 return p
481 }
482 483 // ProxyFromEnvironment returns the URL of the proxy to use for a
484 // given request, as indicated by the environment variables
485 // HTTP_PROXY, HTTPS_PROXY and NO_PROXY (or the lowercase versions
486 // thereof). Requests use the proxy from the environment variable
487 // matching their scheme, unless excluded by NO_PROXY.
488 //
489 // The environment values may be either a complete URL or a
490 // "host[:port]", in which case the "http" scheme is assumed.
491 // An error is returned if the value is a different form.
492 //
493 // A nil URL and nil error are returned if no proxy is defined in the
494 // environment, or a proxy should not be used for the given request,
495 // as defined by NO_PROXY.
496 //
497 // As a special case, if req.URL.Host is "localhost" (with or without
498 // a port number), then a nil URL and nil error will be returned.
499 func ProxyFromEnvironment(req *Request) (*url.URL, error) {
500 return envProxyFunc()(req.URL)
501 }
502 503 // ProxyURL returns a proxy function (for use in a [Transport])
504 // that always returns the same URL.
505 func ProxyURL(fixedURL *url.URL) func(*Request) (*url.URL, error) {
506 return func(*Request) (*url.URL, error) {
507 return fixedURL, nil
508 }
509 }
510 511 // transportRequest is a wrapper around a *Request that adds
512 // optional extra headers to write and stores any error to return
513 // from roundTrip.
514 type transportRequest struct {
515 *Request // original request, not to be mutated
516 extra Header // extra headers to write, or nil
517 trace *httptrace.ClientTrace // optional
518 519 ctx context.Context // canceled when we are done with the request
520 cancel context.CancelCauseFunc
521 522 mu sync.Mutex // guards err
523 err error // first setError value for mapRoundTripError to consider
524 }
525 526 func (tr *transportRequest) extraHeaders() Header {
527 if tr.extra == nil {
528 tr.extra = make(Header)
529 }
530 return tr.extra
531 }
532 533 func (tr *transportRequest) setError(err error) {
534 tr.mu.Lock()
535 if tr.err == nil {
536 tr.err = err
537 }
538 tr.mu.Unlock()
539 }
540 541 // useRegisteredProtocol reports whether an alternate protocol (as registered
542 // with Transport.RegisterProtocol) should be respected for this request.
543 func (t *Transport) useRegisteredProtocol(req *Request) bool {
544 if req.URL.Scheme == "https" && req.requiresHTTP1() {
545 // If this request requires HTTP/1, don't use the
546 // "https" alternate protocol, which is used by the
547 // HTTP/2 code to take over requests if there's an
548 // existing cached HTTP/2 connection.
549 return false
550 }
551 return true
552 }
553 554 // alternateRoundTripper returns the alternate RoundTripper to use
555 // for this request if the Request's URL scheme requires one,
556 // or nil for the normal case of using the Transport.
557 func (t *Transport) alternateRoundTripper(req *Request) RoundTripper {
558 if !t.useRegisteredProtocol(req) {
559 return nil
560 }
561 altProto, _ := t.altProto.Load().(map[string]RoundTripper)
562 return altProto[req.URL.Scheme]
563 }
564 565 func validateHeaders(hdrs Header) string {
566 for k, vv := range hdrs {
567 if !httpguts.ValidHeaderFieldName(k) {
568 return fmt.Sprintf("field name %q", k)
569 }
570 for _, v := range vv {
571 if !httpguts.ValidHeaderFieldValue(v) {
572 // Don't include the value in the error,
573 // because it may be sensitive.
574 return fmt.Sprintf("field value for %q", k)
575 }
576 }
577 }
578 return ""
579 }
580 581 // roundTrip implements a RoundTripper over HTTP.
582 func (t *Transport) roundTrip(req *Request) (_ *Response, err error) {
583 t.nextProtoOnce.Do(t.onceSetNextProtoDefaults)
584 ctx := req.Context()
585 trace := httptrace.ContextClientTrace(ctx)
586 587 if req.URL == nil {
588 req.closeBody()
589 return nil, errors.New("http: nil Request.URL")
590 }
591 if req.Header == nil {
592 req.closeBody()
593 return nil, errors.New("http: nil Request.Header")
594 }
595 scheme := req.URL.Scheme
596 isHTTP := scheme == "http" || scheme == "https"
597 if isHTTP {
598 // Validate the outgoing headers.
599 if err := validateHeaders(req.Header); err != "" {
600 req.closeBody()
601 return nil, fmt.Errorf("net/http: invalid header %s", err)
602 }
603 604 // Validate the outgoing trailers too.
605 if err := validateHeaders(req.Trailer); err != "" {
606 req.closeBody()
607 return nil, fmt.Errorf("net/http: invalid trailer %s", err)
608 }
609 }
610 611 origReq := req
612 req = setupRewindBody(req)
613 614 if altRT := t.alternateRoundTripper(req); altRT != nil {
615 if resp, err := altRT.RoundTrip(req); err != ErrSkipAltProtocol {
616 return resp, err
617 }
618 var err error
619 req, err = rewindBody(req)
620 if err != nil {
621 return nil, err
622 }
623 }
624 if !isHTTP {
625 req.closeBody()
626 return nil, badStringError("unsupported protocol scheme", scheme)
627 }
628 if req.Method != "" && !validMethod(req.Method) {
629 req.closeBody()
630 return nil, fmt.Errorf("net/http: invalid method %q", req.Method)
631 }
632 if req.URL.Host == "" {
633 req.closeBody()
634 return nil, errors.New("http: no Host in request URL")
635 }
636 637 // Transport request context.
638 //
639 // If RoundTrip returns an error, it cancels this context before returning.
640 //
641 // If RoundTrip returns no error:
642 // - For an HTTP/1 request, persistConn.readLoop cancels this context
643 // after reading the request body.
644 // - For an HTTP/2 request, RoundTrip cancels this context after the HTTP/2
645 // RoundTripper returns.
646 ctx, cancel := context.WithCancelCause(req.Context())
647 648 // Convert Request.Cancel into context cancelation.
649 if origReq.Cancel != nil {
650 awaitLegacyCancel(ctx, cancel, origReq)
651 }
652 653 // Convert Transport.CancelRequest into context cancelation.
654 //
655 // This is lamentably expensive. CancelRequest has been deprecated for a long time
656 // and doesn't work on HTTP/2 requests. Perhaps we should drop support for it entirely.
657 cancel = t.prepareTransportCancel(origReq, cancel)
658 659 defer func() {
660 if err != nil {
661 cancel(err)
662 }
663 }()
664 665 for {
666 select {
667 case <-ctx.Done():
668 req.closeBody()
669 return nil, context.Cause(ctx)
670 default:
671 }
672 673 // treq gets modified by roundTrip, so we need to recreate for each retry.
674 treq := &transportRequest{Request: req, trace: trace, ctx: ctx, cancel: cancel}
675 cm, err := t.connectMethodForRequest(treq)
676 if err != nil {
677 req.closeBody()
678 return nil, err
679 }
680 681 // Get the cached or newly-created connection to either the
682 // host (for http or https), the http proxy, or the http proxy
683 // pre-CONNECTed to https server. In any case, we'll be ready
684 // to send it requests.
685 pconn, err := t.getConn(treq, cm)
686 if err != nil {
687 req.closeBody()
688 return nil, err
689 }
690 691 var resp *Response
692 if pconn.alt != nil {
693 // HTTP/2 path.
694 resp, err = pconn.alt.RoundTrip(req)
695 } else {
696 resp, err = pconn.roundTrip(treq)
697 }
698 if err == nil {
699 if pconn.alt != nil {
700 // HTTP/2 requests are not cancelable with CancelRequest,
701 // so we have no further need for the request context.
702 //
703 // On the HTTP/1 path, roundTrip takes responsibility for
704 // canceling the context after the response body is read.
705 cancel(errRequestDone)
706 }
707 resp.Request = origReq
708 return resp, nil
709 }
710 711 // Failed. Clean up and determine whether to retry.
712 if http2isNoCachedConnError(err) {
713 if t.removeIdleConn(pconn) {
714 t.decConnsPerHost(pconn.cacheKey)
715 }
716 } else if !pconn.shouldRetryRequest(req, err) {
717 // Issue 16465: return underlying net.Conn.Read error from peek,
718 // as we've historically done.
719 if e, ok := err.(nothingWrittenError); ok {
720 err = e.error
721 }
722 if e, ok := err.(transportReadFromServerError); ok {
723 err = e.err
724 }
725 if b, ok := req.Body.(*readTrackingBody); ok && !b.didClose {
726 // Issue 49621: Close the request body if pconn.roundTrip
727 // didn't do so already. This can happen if the pconn
728 // write loop exits without reading the write request.
729 req.closeBody()
730 }
731 return nil, err
732 }
733 testHookRoundTripRetried()
734 735 // Rewind the body if we're able to.
736 req, err = rewindBody(req)
737 if err != nil {
738 return nil, err
739 }
740 }
741 }
742 743 func awaitLegacyCancel(ctx context.Context, cancel context.CancelCauseFunc, req *Request) {
744 select {
745 case <-req.Cancel:
746 cancel(errRequestCanceled)
747 case <-ctx.Done():
748 }
749 }
750 751 var errCannotRewind = errors.New("net/http: cannot rewind body after connection loss")
752 753 type readTrackingBody struct {
754 io.ReadCloser
755 didRead bool
756 didClose bool
757 }
758 759 func (r *readTrackingBody) Read(data []byte) (int, error) {
760 r.didRead = true
761 return r.ReadCloser.Read(data)
762 }
763 764 func (r *readTrackingBody) Close() error {
765 r.didClose = true
766 return r.ReadCloser.Close()
767 }
768 769 // setupRewindBody returns a new request with a custom body wrapper
770 // that can report whether the body needs rewinding.
771 // This lets rewindBody avoid an error result when the request
772 // does not have GetBody but the body hasn't been read at all yet.
773 func setupRewindBody(req *Request) *Request {
774 if req.Body == nil || req.Body == NoBody {
775 return req
776 }
777 newReq := *req
778 newReq.Body = &readTrackingBody{ReadCloser: req.Body}
779 return &newReq
780 }
781 782 // rewindBody returns a new request with the body rewound.
783 // It returns req unmodified if the body does not need rewinding.
784 // rewindBody takes care of closing req.Body when appropriate
785 // (in all cases except when rewindBody returns req unmodified).
786 func rewindBody(req *Request) (rewound *Request, err error) {
787 if req.Body == nil || req.Body == NoBody || (!req.Body.(*readTrackingBody).didRead && !req.Body.(*readTrackingBody).didClose) {
788 return req, nil // nothing to rewind
789 }
790 if !req.Body.(*readTrackingBody).didClose {
791 req.closeBody()
792 }
793 if req.GetBody == nil {
794 return nil, errCannotRewind
795 }
796 body, err := req.GetBody()
797 if err != nil {
798 return nil, err
799 }
800 newReq := *req
801 newReq.Body = &readTrackingBody{ReadCloser: body}
802 return &newReq, nil
803 }
804 805 // shouldRetryRequest reports whether we should retry sending a failed
806 // HTTP request on a new connection. The non-nil input error is the
807 // error from roundTrip.
808 func (pc *persistConn) shouldRetryRequest(req *Request, err error) bool {
809 if http2isNoCachedConnError(err) {
810 // Issue 16582: if the user started a bunch of
811 // requests at once, they can all pick the same conn
812 // and violate the server's max concurrent streams.
813 // Instead, match the HTTP/1 behavior for now and dial
814 // again to get a new TCP connection, rather than failing
815 // this request.
816 return true
817 }
818 if err == errMissingHost {
819 // User error.
820 return false
821 }
822 if !pc.isReused() {
823 // This was a fresh connection. There's no reason the server
824 // should've hung up on us.
825 //
826 // Also, if we retried now, we could loop forever
827 // creating new connections and retrying if the server
828 // is just hanging up on us because it doesn't like
829 // our request (as opposed to sending an error).
830 return false
831 }
832 if _, ok := err.(nothingWrittenError); ok {
833 // We never wrote anything, so it's safe to retry, if there's no body or we
834 // can "rewind" the body with GetBody.
835 return req.outgoingLength() == 0 || req.GetBody != nil
836 }
837 if !req.isReplayable() {
838 // Don't retry non-idempotent requests.
839 return false
840 }
841 if _, ok := err.(transportReadFromServerError); ok {
842 // We got some non-EOF net.Conn.Read failure reading
843 // the 1st response byte from the server.
844 return true
845 }
846 if err == errServerClosedIdle {
847 // The server replied with io.EOF while we were trying to
848 // read the response. Probably an unfortunately keep-alive
849 // timeout, just as the client was writing a request.
850 return true
851 }
852 return false // conservatively
853 }
854 855 // ErrSkipAltProtocol is a sentinel error value defined by Transport.RegisterProtocol.
856 var ErrSkipAltProtocol = errors.New("net/http: skip alternate protocol")
857 858 // RegisterProtocol registers a new protocol with scheme.
859 // The [Transport] will pass requests using the given scheme to rt.
860 // It is rt's responsibility to simulate HTTP request semantics.
861 //
862 // RegisterProtocol can be used by other packages to provide
863 // implementations of protocol schemes like "ftp" or "file".
864 //
865 // If rt.RoundTrip returns [ErrSkipAltProtocol], the Transport will
866 // handle the [Transport.RoundTrip] itself for that one request, as if the
867 // protocol were not registered.
868 func (t *Transport) RegisterProtocol(scheme string, rt RoundTripper) {
869 t.altMu.Lock()
870 defer t.altMu.Unlock()
871 oldMap, _ := t.altProto.Load().(map[string]RoundTripper)
872 if _, exists := oldMap[scheme]; exists {
873 panic("protocol " + scheme + " already registered")
874 }
875 newMap := maps.Clone(oldMap)
876 if newMap == nil {
877 newMap = map[string]RoundTripper{}
878 }
879 newMap[scheme] = rt
880 t.altProto.Store(newMap)
881 }
882 883 // CloseIdleConnections closes any connections which were previously
884 // connected from previous requests but are now sitting idle in
885 // a "keep-alive" state. It does not interrupt any connections currently
886 // in use.
887 func (t *Transport) CloseIdleConnections() {
888 t.nextProtoOnce.Do(t.onceSetNextProtoDefaults)
889 t.idleMu.Lock()
890 m := t.idleConn
891 t.idleConn = nil
892 t.closeIdle = true // close newly idle connections
893 t.idleLRU = connLRU{}
894 t.idleMu.Unlock()
895 for _, conns := range m {
896 for _, pconn := range conns {
897 pconn.close(errCloseIdleConns)
898 }
899 }
900 t.connsPerHostMu.Lock()
901 t.dialsInProgress.all(func(w *wantConn) {
902 if w.cancelCtx != nil && !w.waiting() {
903 w.cancelCtx()
904 }
905 })
906 t.connsPerHostMu.Unlock()
907 if t2 := t.h2transport; t2 != nil {
908 t2.CloseIdleConnections()
909 }
910 }
911 912 // prepareTransportCancel sets up state to convert Transport.CancelRequest into context cancelation.
913 func (t *Transport) prepareTransportCancel(req *Request, origCancel context.CancelCauseFunc) context.CancelCauseFunc {
914 // Historically, RoundTrip has not modified the Request in any way.
915 // We could avoid the need to keep a map of all in-flight requests by adding
916 // a field to the Request containing its cancel func, and setting that field
917 // while the request is in-flight. Callers aren't supposed to reuse a Request
918 // until after the response body is closed, so this wouldn't violate any
919 // concurrency guarantees.
920 cancel := func(err error) {
921 origCancel(err)
922 t.reqMu.Lock()
923 delete(t.reqCanceler, req)
924 t.reqMu.Unlock()
925 }
926 t.reqMu.Lock()
927 if t.reqCanceler == nil {
928 t.reqCanceler = map[*Request]context.CancelCauseFunc{}
929 }
930 t.reqCanceler[req] = cancel
931 t.reqMu.Unlock()
932 return cancel
933 }
934 935 // CancelRequest cancels an in-flight request by closing its connection.
936 // CancelRequest should only be called after [Transport.RoundTrip] has returned.
937 //
938 // Deprecated: Use [Request.WithContext] to create a request with a
939 // cancelable context instead. CancelRequest cannot cancel HTTP/2
940 // requests. This may become a no-op in a future release of Go.
941 func (t *Transport) CancelRequest(req *Request) {
942 t.reqMu.Lock()
943 cancel := t.reqCanceler[req]
944 t.reqMu.Unlock()
945 if cancel != nil {
946 cancel(errRequestCanceled)
947 }
948 }
949 950 //
951 // Private implementation past this point.
952 //
953 954 var (
955 envProxyOnce sync.Once
956 envProxyFuncValue func(*url.URL) (*url.URL, error)
957 )
958 959 // envProxyFunc returns a function that reads the
960 // environment variable to determine the proxy address.
961 func envProxyFunc() func(*url.URL) (*url.URL, error) {
962 envProxyOnce.Do(func() {
963 envProxyFuncValue = httpproxy.FromEnvironment().ProxyFunc()
964 })
965 return envProxyFuncValue
966 }
967 968 // resetProxyConfig is used by tests.
969 func resetProxyConfig() {
970 envProxyOnce = sync.Once{}
971 envProxyFuncValue = nil
972 }
973 974 func (t *Transport) connectMethodForRequest(treq *transportRequest) (cm connectMethod, err error) {
975 cm.targetScheme = treq.URL.Scheme
976 cm.targetAddr = canonicalAddr(treq.URL)
977 if t.Proxy != nil {
978 cm.proxyURL, err = t.Proxy(treq.Request)
979 }
980 cm.onlyH1 = treq.requiresHTTP1()
981 return cm, err
982 }
983 984 // proxyAuth returns the Proxy-Authorization header to set
985 // on requests, if applicable.
986 func (cm *connectMethod) proxyAuth() string {
987 if cm.proxyURL == nil {
988 return ""
989 }
990 if u := cm.proxyURL.User; u != nil {
991 username := u.Username()
992 password, _ := u.Password()
993 return "Basic " + basicAuth(username, password)
994 }
995 return ""
996 }
997 998 // error values for debugging and testing, not seen by users.
999 var (
1000 errKeepAlivesDisabled = errors.New("http: putIdleConn: keep alives disabled")
1001 errConnBroken = errors.New("http: putIdleConn: connection is in bad state")
1002 errCloseIdle = errors.New("http: putIdleConn: CloseIdleConnections was called")
1003 errTooManyIdle = errors.New("http: putIdleConn: too many idle connections")
1004 errTooManyIdleHost = errors.New("http: putIdleConn: too many idle connections for host")
1005 errCloseIdleConns = errors.New("http: CloseIdleConnections called")
1006 errReadLoopExiting = errors.New("http: persistConn.readLoop exiting")
1007 errIdleConnTimeout = errors.New("http: idle connection timeout")
1008 1009 // errServerClosedIdle is not seen by users for idempotent requests, but may be
1010 // seen by a user if the server shuts down an idle connection and sends its FIN
1011 // in flight with already-written POST body bytes from the client.
1012 // See https://github.com/golang/go/issues/19943#issuecomment-355607646
1013 errServerClosedIdle = errors.New("http: server closed idle connection")
1014 )
1015 1016 // transportReadFromServerError is used by Transport.readLoop when the
1017 // 1 byte peek read fails and we're actually anticipating a response.
1018 // Usually this is just due to the inherent keep-alive shut down race,
1019 // where the server closed the connection at the same time the client
1020 // wrote. The underlying err field is usually io.EOF or some
1021 // ECONNRESET sort of thing which varies by platform. But it might be
1022 // the user's custom net.Conn.Read error too, so we carry it along for
1023 // them to return from Transport.RoundTrip.
1024 type transportReadFromServerError struct {
1025 err error
1026 }
1027 1028 func (e transportReadFromServerError) Unwrap() error { return e.err }
1029 1030 func (e transportReadFromServerError) Error() string {
1031 return fmt.Sprintf("net/http: Transport failed to read from server: %v", e.err)
1032 }
1033 1034 func (t *Transport) putOrCloseIdleConn(pconn *persistConn) {
1035 if err := t.tryPutIdleConn(pconn); err != nil {
1036 pconn.close(err)
1037 }
1038 }
1039 1040 func (t *Transport) maxIdleConnsPerHost() int {
1041 if v := t.MaxIdleConnsPerHost; v != 0 {
1042 return v
1043 }
1044 return DefaultMaxIdleConnsPerHost
1045 }
1046 1047 // tryPutIdleConn adds pconn to the list of idle persistent connections awaiting
1048 // a new request.
1049 // If pconn is no longer needed or not in a good state, tryPutIdleConn returns
1050 // an error explaining why it wasn't registered.
1051 // tryPutIdleConn does not close pconn. Use putOrCloseIdleConn instead for that.
1052 func (t *Transport) tryPutIdleConn(pconn *persistConn) error {
1053 if t.DisableKeepAlives || t.MaxIdleConnsPerHost < 0 {
1054 return errKeepAlivesDisabled
1055 }
1056 if pconn.isBroken() {
1057 return errConnBroken
1058 }
1059 pconn.markReused()
1060 1061 t.idleMu.Lock()
1062 defer t.idleMu.Unlock()
1063 1064 // HTTP/2 (pconn.alt != nil) connections do not come out of the idle list,
1065 // because multiple goroutines can use them simultaneously.
1066 // If this is an HTTP/2 connection being “returned,” we're done.
1067 if pconn.alt != nil && t.idleLRU.m[pconn] != nil {
1068 return nil
1069 }
1070 1071 // Deliver pconn to goroutine waiting for idle connection, if any.
1072 // (They may be actively dialing, but this conn is ready first.
1073 // Chrome calls this socket late binding.
1074 // See https://www.chromium.org/developers/design-documents/network-stack#TOC-Connection-Management.)
1075 key := pconn.cacheKey
1076 if q, ok := t.idleConnWait[key]; ok {
1077 done := false
1078 if pconn.alt == nil {
1079 // HTTP/1.
1080 // Loop over the waiting list until we find a w that isn't done already, and hand it pconn.
1081 for q.len() > 0 {
1082 w := q.popFront()
1083 if w.tryDeliver(pconn, nil, time.Time{}) {
1084 done = true
1085 break
1086 }
1087 }
1088 } else {
1089 // HTTP/2.
1090 // Can hand the same pconn to everyone in the waiting list,
1091 // and we still won't be done: we want to put it in the idle
1092 // list unconditionally, for any future clients too.
1093 for q.len() > 0 {
1094 w := q.popFront()
1095 w.tryDeliver(pconn, nil, time.Time{})
1096 }
1097 }
1098 if q.len() == 0 {
1099 delete(t.idleConnWait, key)
1100 } else {
1101 t.idleConnWait[key] = q
1102 }
1103 if done {
1104 return nil
1105 }
1106 }
1107 1108 if t.closeIdle {
1109 return errCloseIdle
1110 }
1111 if t.idleConn == nil {
1112 t.idleConn = map[connectMethodKey][]*persistConn{}
1113 }
1114 idles := t.idleConn[key]
1115 if len(idles) >= t.maxIdleConnsPerHost() {
1116 return errTooManyIdleHost
1117 }
1118 for _, exist := range idles {
1119 if exist == pconn {
1120 log.Fatalf("dup idle pconn %p in freelist", pconn)
1121 }
1122 }
1123 t.idleConn[key] = append(idles, pconn)
1124 t.idleLRU.add(pconn)
1125 if t.MaxIdleConns != 0 && t.idleLRU.len() > t.MaxIdleConns {
1126 oldest := t.idleLRU.removeOldest()
1127 oldest.close(errTooManyIdle)
1128 t.removeIdleConnLocked(oldest)
1129 }
1130 1131 // Set idle timer, but only for HTTP/1 (pconn.alt == nil).
1132 // The HTTP/2 implementation manages the idle timer itself
1133 // (see idleConnTimeout in h2_bundle.go).
1134 if t.IdleConnTimeout > 0 && pconn.alt == nil {
1135 if pconn.idleTimer != nil {
1136 pconn.idleTimer.Reset(t.IdleConnTimeout)
1137 } else {
1138 pconn.idleTimer = time.AfterFunc(t.IdleConnTimeout, pconn.closeConnIfStillIdle)
1139 }
1140 }
1141 pconn.idleAt = time.Now()
1142 return nil
1143 }
1144 1145 // queueForIdleConn queues w to receive the next idle connection for w.cm.
1146 // As an optimization hint to the caller, queueForIdleConn reports whether
1147 // it successfully delivered an already-idle connection.
1148 func (t *Transport) queueForIdleConn(w *wantConn) (delivered bool) {
1149 if t.DisableKeepAlives {
1150 return false
1151 }
1152 1153 t.idleMu.Lock()
1154 defer t.idleMu.Unlock()
1155 1156 // Stop closing connections that become idle - we might want one.
1157 // (That is, undo the effect of t.CloseIdleConnections.)
1158 t.closeIdle = false
1159 1160 if w == nil {
1161 // Happens in test hook.
1162 return false
1163 }
1164 1165 // If IdleConnTimeout is set, calculate the oldest
1166 // persistConn.idleAt time we're willing to use a cached idle
1167 // conn.
1168 var oldTime time.Time
1169 if t.IdleConnTimeout > 0 {
1170 oldTime = time.Now().Add(-t.IdleConnTimeout)
1171 }
1172 1173 // Look for most recently-used idle connection.
1174 if list, ok := t.idleConn[w.key]; ok {
1175 stop := false
1176 delivered := false
1177 for len(list) > 0 && !stop {
1178 pconn := list[len(list)-1]
1179 1180 // See whether this connection has been idle too long, considering
1181 // only the wall time (the Round(0)), in case this is a laptop or VM
1182 // coming out of suspend with previously cached idle connections.
1183 tooOld := !oldTime.IsZero() && pconn.idleAt.Round(0).Before(oldTime)
1184 if tooOld {
1185 // Async cleanup. Launch in its own goroutine (as if a
1186 // time.AfterFunc called it); it acquires idleMu, which we're
1187 // holding, and does a synchronous net.Conn.Close.
1188 pconn.closeConnIfStillIdle()
1189 }
1190 if pconn.isBroken() || tooOld {
1191 // If either persistConn.readLoop has marked the connection
1192 // broken, but Transport.removeIdleConn has not yet removed it
1193 // from the idle list, or if this persistConn is too old (it was
1194 // idle too long), then ignore it and look for another. In both
1195 // cases it's already in the process of being closed.
1196 list = list[:len(list)-1]
1197 continue
1198 }
1199 delivered = w.tryDeliver(pconn, nil, pconn.idleAt)
1200 if delivered {
1201 if pconn.alt != nil {
1202 // HTTP/2: multiple clients can share pconn.
1203 // Leave it in the list.
1204 } else {
1205 // HTTP/1: only one client can use pconn.
1206 // Remove it from the list.
1207 t.idleLRU.remove(pconn)
1208 list = list[:len(list)-1]
1209 }
1210 }
1211 stop = true
1212 }
1213 if len(list) > 0 {
1214 t.idleConn[w.key] = list
1215 } else {
1216 delete(t.idleConn, w.key)
1217 }
1218 if stop {
1219 return delivered
1220 }
1221 }
1222 1223 // Register to receive next connection that becomes idle.
1224 if t.idleConnWait == nil {
1225 t.idleConnWait = map[connectMethodKey]wantConnQueue{}
1226 }
1227 q := t.idleConnWait[w.key]
1228 q.cleanFrontNotWaiting()
1229 q.pushBack(w)
1230 t.idleConnWait[w.key] = q
1231 return false
1232 }
1233 1234 // removeIdleConn marks pconn as dead.
1235 func (t *Transport) removeIdleConn(pconn *persistConn) bool {
1236 t.idleMu.Lock()
1237 defer t.idleMu.Unlock()
1238 return t.removeIdleConnLocked(pconn)
1239 }
1240 1241 // t.idleMu must be held.
1242 func (t *Transport) removeIdleConnLocked(pconn *persistConn) bool {
1243 if pconn.idleTimer != nil {
1244 pconn.idleTimer.Stop()
1245 }
1246 t.idleLRU.remove(pconn)
1247 key := pconn.cacheKey
1248 pconns := t.idleConn[key]
1249 var removed bool
1250 switch len(pconns) {
1251 case 0:
1252 // Nothing
1253 case 1:
1254 if pconns[0] == pconn {
1255 delete(t.idleConn, key)
1256 removed = true
1257 }
1258 default:
1259 for i, v := range pconns {
1260 if v != pconn {
1261 continue
1262 }
1263 // Slide down, keeping most recently-used
1264 // conns at the end.
1265 copy(pconns[i:], pconns[i+1:])
1266 t.idleConn[key] = pconns[:len(pconns)-1]
1267 removed = true
1268 break
1269 }
1270 }
1271 return removed
1272 }
1273 1274 var zeroDialer net.Dialer
1275 1276 func (t *Transport) dial(ctx context.Context, network, addr string) (net.Conn, error) {
1277 if t.DialContext != nil {
1278 c, err := t.DialContext(ctx, network, addr)
1279 if c == nil && err == nil {
1280 err = errors.New("net/http: Transport.DialContext hook returned (nil, nil)")
1281 }
1282 return c, err
1283 }
1284 if t.Dial != nil {
1285 c, err := t.Dial(network, addr)
1286 if c == nil && err == nil {
1287 err = errors.New("net/http: Transport.Dial hook returned (nil, nil)")
1288 }
1289 return c, err
1290 }
1291 return zeroDialer.DialContext(ctx, network, addr)
1292 }
1293 1294 // A wantConn records state about a wanted connection
1295 // (that is, an active call to getConn).
1296 // The conn may be gotten by dialing or by finding an idle connection,
1297 // or a cancellation may make the conn no longer wanted.
1298 // These three options are racing against each other and use
1299 // wantConn to coordinate and agree about the winning outcome.
1300 type wantConn struct {
1301 cm connectMethod
1302 key connectMethodKey // cm.key()
1303 1304 // hooks for testing to know when dials are done
1305 // beforeDial is called in the getConn goroutine when the dial is queued.
1306 // afterDial is called when the dial is completed or canceled.
1307 beforeDial func()
1308 afterDial func()
1309 1310 mu sync.Mutex // protects ctx, done and sending of the result
1311 ctx context.Context // context for dial, cleared after delivered or canceled
1312 cancelCtx context.CancelFunc
1313 done bool // true after delivered or canceled
1314 result chan connOrError // channel to deliver connection or error
1315 }
1316 1317 type connOrError struct {
1318 pc *persistConn
1319 err error
1320 idleAt time.Time
1321 }
1322 1323 // waiting reports whether w is still waiting for an answer (connection or error).
1324 func (w *wantConn) waiting() bool {
1325 w.mu.Lock()
1326 defer w.mu.Unlock()
1327 1328 return !w.done
1329 }
1330 1331 // getCtxForDial returns context for dial or nil if connection was delivered or canceled.
1332 func (w *wantConn) getCtxForDial() context.Context {
1333 w.mu.Lock()
1334 defer w.mu.Unlock()
1335 1336 return w.ctx
1337 }
1338 1339 // tryDeliver attempts to deliver pc, err to w and reports whether it succeeded.
1340 func (w *wantConn) tryDeliver(pc *persistConn, err error, idleAt time.Time) bool {
1341 w.mu.Lock()
1342 defer w.mu.Unlock()
1343 1344 if w.done {
1345 return false
1346 }
1347 if (pc == nil) == (err == nil) {
1348 panic("net/http: internal error: misuse of tryDeliver")
1349 }
1350 w.ctx = nil
1351 w.done = true
1352 1353 w.result <- connOrError{pc: pc, err: err, idleAt: idleAt}
1354 close(w.result)
1355 1356 return true
1357 }
1358 1359 // cancel marks w as no longer wanting a result (for example, due to cancellation).
1360 // If a connection has been delivered already, cancel returns it with t.putOrCloseIdleConn.
1361 func (w *wantConn) cancel(t *Transport) {
1362 w.mu.Lock()
1363 var pc *persistConn
1364 if w.done {
1365 if r, ok := <-w.result; ok {
1366 pc = r.pc
1367 }
1368 } else {
1369 close(w.result)
1370 }
1371 w.ctx = nil
1372 w.done = true
1373 w.mu.Unlock()
1374 1375 // HTTP/2 connections (pc.alt != nil) aren't removed from the idle pool on use,
1376 // and should not be added back here. If the pconn isn't in the idle pool,
1377 // it's because we removed it due to an error.
1378 if pc != nil && pc.alt == nil {
1379 t.putOrCloseIdleConn(pc)
1380 }
1381 }
1382 1383 // A wantConnQueue is a queue of wantConns.
1384 type wantConnQueue struct {
1385 // This is a queue, not a deque.
1386 // It is split into two stages - head[headPos:] and tail.
1387 // popFront is trivial (headPos++) on the first stage, and
1388 // pushBack is trivial (append) on the second stage.
1389 // If the first stage is empty, popFront can swap the
1390 // first and second stages to remedy the situation.
1391 //
1392 // This two-stage split is analogous to the use of two lists
1393 // in Okasaki's purely functional queue but without the
1394 // overhead of reversing the list when swapping stages.
1395 head []*wantConn
1396 headPos int
1397 tail []*wantConn
1398 }
1399 1400 // len returns the number of items in the queue.
1401 func (q *wantConnQueue) len() int {
1402 return len(q.head) - q.headPos + len(q.tail)
1403 }
1404 1405 // pushBack adds w to the back of the queue.
1406 func (q *wantConnQueue) pushBack(w *wantConn) {
1407 q.tail = append(q.tail, w)
1408 }
1409 1410 // popFront removes and returns the wantConn at the front of the queue.
1411 func (q *wantConnQueue) popFront() *wantConn {
1412 if q.headPos >= len(q.head) {
1413 if len(q.tail) == 0 {
1414 return nil
1415 }
1416 // Pick up tail as new head, clear tail.
1417 q.head, q.headPos, q.tail = q.tail, 0, q.head[:0]
1418 }
1419 w := q.head[q.headPos]
1420 q.head[q.headPos] = nil
1421 q.headPos++
1422 return w
1423 }
1424 1425 // peekFront returns the wantConn at the front of the queue without removing it.
1426 func (q *wantConnQueue) peekFront() *wantConn {
1427 if q.headPos < len(q.head) {
1428 return q.head[q.headPos]
1429 }
1430 if len(q.tail) > 0 {
1431 return q.tail[0]
1432 }
1433 return nil
1434 }
1435 1436 // cleanFrontNotWaiting pops any wantConns that are no longer waiting from the head of the
1437 // queue, reporting whether any were popped.
1438 func (q *wantConnQueue) cleanFrontNotWaiting() (cleaned bool) {
1439 for {
1440 w := q.peekFront()
1441 if w == nil || w.waiting() {
1442 return cleaned
1443 }
1444 q.popFront()
1445 cleaned = true
1446 }
1447 }
1448 1449 // cleanFrontCanceled pops any wantConns with canceled dials from the head of the queue.
1450 func (q *wantConnQueue) cleanFrontCanceled() {
1451 for {
1452 w := q.peekFront()
1453 if w == nil || w.cancelCtx != nil {
1454 return
1455 }
1456 q.popFront()
1457 }
1458 }
1459 1460 // all iterates over all wantConns in the queue.
1461 // The caller must not modify the queue while iterating.
1462 func (q *wantConnQueue) all(f func(*wantConn)) {
1463 for _, w := range q.head[q.headPos:] {
1464 f(w)
1465 }
1466 for _, w := range q.tail {
1467 f(w)
1468 }
1469 }
1470 1471 func (t *Transport) customDialTLS(ctx context.Context, network, addr string) (conn net.Conn, err error) {
1472 if t.DialTLSContext != nil {
1473 conn, err = t.DialTLSContext(ctx, network, addr)
1474 } else {
1475 conn, err = t.DialTLS(network, addr)
1476 }
1477 if conn == nil && err == nil {
1478 err = errors.New("net/http: Transport.DialTLS or DialTLSContext returned (nil, nil)")
1479 }
1480 return
1481 }
1482 1483 // getConn dials and creates a new persistConn to the target as
1484 // specified in the connectMethod. This includes doing a proxy CONNECT
1485 // and/or setting up TLS. If this doesn't return an error, the persistConn
1486 // is ready to write requests to.
1487 func (t *Transport) getConn(treq *transportRequest, cm connectMethod) (_ *persistConn, err error) {
1488 req := treq.Request
1489 trace := treq.trace
1490 ctx := req.Context()
1491 if trace != nil && trace.GetConn != nil {
1492 trace.GetConn(cm.addr())
1493 }
1494 1495 // Detach from the request context's cancellation signal.
1496 // The dial should proceed even if the request is canceled,
1497 // because a future request may be able to make use of the connection.
1498 //
1499 // We retain the request context's values.
1500 dialCtx, dialCancel := context.WithCancel(context.WithoutCancel(ctx))
1501 1502 w := &wantConn{
1503 cm: cm,
1504 key: cm.key(),
1505 ctx: dialCtx,
1506 cancelCtx: dialCancel,
1507 result: chan connOrError{1},
1508 beforeDial: testHookPrePendingDial,
1509 afterDial: testHookPostPendingDial,
1510 }
1511 defer func() {
1512 if err != nil {
1513 w.cancel(t)
1514 }
1515 }()
1516 1517 // Queue for idle connection.
1518 if delivered := t.queueForIdleConn(w); !delivered {
1519 t.queueForDial(w)
1520 }
1521 1522 // Wait for completion or cancellation.
1523 select {
1524 case r := <-w.result:
1525 // Trace success but only for HTTP/1.
1526 // HTTP/2 calls trace.GotConn itself.
1527 if r.pc != nil && r.pc.alt == nil && trace != nil && trace.GotConn != nil {
1528 info := httptrace.GotConnInfo{
1529 Conn: r.pc.conn,
1530 Reused: r.pc.isReused(),
1531 }
1532 if !r.idleAt.IsZero() {
1533 info.WasIdle = true
1534 info.IdleTime = time.Since(r.idleAt)
1535 }
1536 trace.GotConn(info)
1537 }
1538 if r.err != nil {
1539 // If the request has been canceled, that's probably
1540 // what caused r.err; if so, prefer to return the
1541 // cancellation error (see golang.org/issue/16049).
1542 select {
1543 case <-treq.ctx.Done():
1544 err := context.Cause(treq.ctx)
1545 if err == errRequestCanceled {
1546 err = errRequestCanceledConn
1547 }
1548 return nil, err
1549 default:
1550 // return below
1551 }
1552 }
1553 return r.pc, r.err
1554 case <-treq.ctx.Done():
1555 err := context.Cause(treq.ctx)
1556 if err == errRequestCanceled {
1557 err = errRequestCanceledConn
1558 }
1559 return nil, err
1560 }
1561 }
1562 1563 // queueForDial queues w to wait for permission to begin dialing.
1564 // Once w receives permission to dial, it will do so in a separate goroutine.
1565 func (t *Transport) queueForDial(w *wantConn) {
1566 w.beforeDial()
1567 1568 t.connsPerHostMu.Lock()
1569 defer t.connsPerHostMu.Unlock()
1570 1571 if t.MaxConnsPerHost <= 0 {
1572 t.startDialConnForLocked(w)
1573 return
1574 }
1575 1576 if n := t.connsPerHost[w.key]; n < t.MaxConnsPerHost {
1577 if t.connsPerHost == nil {
1578 t.connsPerHost = map[connectMethodKey]int{}
1579 }
1580 t.connsPerHost[w.key] = n + 1
1581 t.startDialConnForLocked(w)
1582 return
1583 }
1584 1585 if t.connsPerHostWait == nil {
1586 t.connsPerHostWait = map[connectMethodKey]wantConnQueue{}
1587 }
1588 q := t.connsPerHostWait[w.key]
1589 q.cleanFrontNotWaiting()
1590 q.pushBack(w)
1591 t.connsPerHostWait[w.key] = q
1592 }
1593 1594 // startDialConnFor calls dialConn in a new goroutine.
1595 // t.connsPerHostMu must be held.
1596 func (t *Transport) startDialConnForLocked(w *wantConn) {
1597 t.dialsInProgress.cleanFrontCanceled()
1598 t.dialsInProgress.pushBack(w)
1599 func() {
1600 t.dialConnFor(w)
1601 t.connsPerHostMu.Lock()
1602 defer t.connsPerHostMu.Unlock()
1603 w.cancelCtx = nil
1604 }()
1605 }
1606 1607 // dialConnFor dials on behalf of w and delivers the result to w.
1608 // dialConnFor has received permission to dial w.cm and is counted in t.connCount[w.cm.key()].
1609 // If the dial is canceled or unsuccessful, dialConnFor decrements t.connCount[w.cm.key()].
1610 func (t *Transport) dialConnFor(w *wantConn) {
1611 defer w.afterDial()
1612 ctx := w.getCtxForDial()
1613 if ctx == nil {
1614 t.decConnsPerHost(w.key)
1615 return
1616 }
1617 1618 pc, err := t.dialConn(ctx, w.cm)
1619 delivered := w.tryDeliver(pc, err, time.Time{})
1620 if err == nil && (!delivered || pc.alt != nil) {
1621 // pconn was not passed to w,
1622 // or it is HTTP/2 and can be shared.
1623 // Add to the idle connection pool.
1624 t.putOrCloseIdleConn(pc)
1625 }
1626 if err != nil {
1627 t.decConnsPerHost(w.key)
1628 }
1629 }
1630 1631 // decConnsPerHost decrements the per-host connection count for key,
1632 // which may in turn give a different waiting goroutine permission to dial.
1633 func (t *Transport) decConnsPerHost(key connectMethodKey) {
1634 if t.MaxConnsPerHost <= 0 {
1635 return
1636 }
1637 1638 t.connsPerHostMu.Lock()
1639 defer t.connsPerHostMu.Unlock()
1640 n := t.connsPerHost[key]
1641 if n == 0 {
1642 // Shouldn't happen, but if it does, the counting is buggy and could
1643 // easily lead to a silent deadlock, so report the problem loudly.
1644 panic("net/http: internal error: connCount underflow")
1645 }
1646 1647 // Can we hand this count to a goroutine still waiting to dial?
1648 // (Some goroutines on the wait list may have timed out or
1649 // gotten a connection another way. If they're all gone,
1650 // we don't want to kick off any spurious dial operations.)
1651 if q := t.connsPerHostWait[key]; q.len() > 0 {
1652 done := false
1653 for q.len() > 0 {
1654 w := q.popFront()
1655 if w.waiting() {
1656 t.startDialConnForLocked(w)
1657 done = true
1658 break
1659 }
1660 }
1661 if q.len() == 0 {
1662 delete(t.connsPerHostWait, key)
1663 } else {
1664 // q is a value (like a slice), so we have to store
1665 // the updated q back into the map.
1666 t.connsPerHostWait[key] = q
1667 }
1668 if done {
1669 return
1670 }
1671 }
1672 1673 // Otherwise, decrement the recorded count.
1674 if n--; n == 0 {
1675 delete(t.connsPerHost, key)
1676 } else {
1677 t.connsPerHost[key] = n
1678 }
1679 }
1680 1681 // Add TLS to a persistent connection, i.e. negotiate a TLS session. If pconn is already a TLS
1682 // tunnel, this function establishes a nested TLS session inside the encrypted channel.
1683 // The remote endpoint's name may be overridden by TLSClientConfig.ServerName.
1684 func (pconn *persistConn) addTLS(ctx context.Context, name string, trace *httptrace.ClientTrace) error {
1685 // Initiate TLS and check remote host name against certificate.
1686 cfg := cloneTLSConfig(pconn.t.TLSClientConfig)
1687 if cfg.ServerName == "" {
1688 cfg.ServerName = name
1689 }
1690 if pconn.cacheKey.onlyH1 {
1691 cfg.NextProtos = nil
1692 }
1693 plainConn := pconn.conn
1694 tlsConn := tls.Client(plainConn, cfg)
1695 errc := chan error{2}
1696 var timer *time.Timer // for canceling TLS handshake
1697 if d := pconn.t.TLSHandshakeTimeout; d != 0 {
1698 timer = time.AfterFunc(d, func() {
1699 errc <- tlsHandshakeTimeoutError{}
1700 })
1701 }
1702 func() {
1703 if trace != nil && trace.TLSHandshakeStart != nil {
1704 trace.TLSHandshakeStart()
1705 }
1706 err := tlsConn.HandshakeContext(ctx)
1707 if timer != nil {
1708 timer.Stop()
1709 }
1710 errc <- err
1711 }()
1712 if err := <-errc; err != nil {
1713 plainConn.Close()
1714 if err == (tlsHandshakeTimeoutError{}) {
1715 // Now that we have closed the connection,
1716 // wait for the call to HandshakeContext to return.
1717 <-errc
1718 }
1719 if trace != nil && trace.TLSHandshakeDone != nil {
1720 trace.TLSHandshakeDone(tls.ConnectionState{}, err)
1721 }
1722 return err
1723 }
1724 cs := tlsConn.ConnectionState()
1725 if trace != nil && trace.TLSHandshakeDone != nil {
1726 trace.TLSHandshakeDone(cs, nil)
1727 }
1728 pconn.tlsState = &cs
1729 pconn.conn = tlsConn
1730 return nil
1731 }
1732 1733 type erringRoundTripper interface {
1734 RoundTripErr() error
1735 }
1736 1737 var testHookProxyConnectTimeout = context.WithTimeout
1738 1739 func (t *Transport) dialConn(ctx context.Context, cm connectMethod) (pconn *persistConn, err error) {
1740 pconn = &persistConn{
1741 t: t,
1742 cacheKey: cm.key(),
1743 reqch: chan requestAndChan{1},
1744 writech: chan writeRequest{1},
1745 closech: chan struct{}{},
1746 writeErrCh: chan error{1},
1747 writeLoopDone: chan struct{}{},
1748 }
1749 trace := httptrace.ContextClientTrace(ctx)
1750 wrapErr := func(err error) error {
1751 if cm.proxyURL != nil {
1752 // Return a typed error, per Issue 16997
1753 return &net.OpError{Op: "proxyconnect", Net: "tcp", Err: err}
1754 }
1755 return err
1756 }
1757 if cm.scheme() == "https" && t.hasCustomTLSDialer() {
1758 var err error
1759 pconn.conn, err = t.customDialTLS(ctx, "tcp", cm.addr())
1760 if err != nil {
1761 return nil, wrapErr(err)
1762 }
1763 if tc, ok := pconn.conn.(*tls.Conn); ok {
1764 // Handshake here, in case DialTLS didn't. TLSNextProto below
1765 // depends on it for knowing the connection state.
1766 if trace != nil && trace.TLSHandshakeStart != nil {
1767 trace.TLSHandshakeStart()
1768 }
1769 if err := tc.HandshakeContext(ctx); err != nil {
1770 pconn.conn.Close()
1771 if trace != nil && trace.TLSHandshakeDone != nil {
1772 trace.TLSHandshakeDone(tls.ConnectionState{}, err)
1773 }
1774 return nil, err
1775 }
1776 cs := tc.ConnectionState()
1777 if trace != nil && trace.TLSHandshakeDone != nil {
1778 trace.TLSHandshakeDone(cs, nil)
1779 }
1780 pconn.tlsState = &cs
1781 }
1782 } else {
1783 conn, err := t.dial(ctx, "tcp", cm.addr())
1784 if err != nil {
1785 return nil, wrapErr(err)
1786 }
1787 pconn.conn = conn
1788 if cm.scheme() == "https" {
1789 var firstTLSHost string
1790 if firstTLSHost, _, err = net.SplitHostPort(cm.addr()); err != nil {
1791 return nil, wrapErr(err)
1792 }
1793 if err = pconn.addTLS(ctx, firstTLSHost, trace); err != nil {
1794 return nil, wrapErr(err)
1795 }
1796 }
1797 }
1798 1799 // Proxy setup.
1800 switch {
1801 case cm.proxyURL == nil:
1802 // Do nothing. Not using a proxy.
1803 case cm.proxyURL.Scheme == "socks5" || cm.proxyURL.Scheme == "socks5h":
1804 conn := pconn.conn
1805 d := socksNewDialer("tcp", conn.RemoteAddr().String())
1806 if u := cm.proxyURL.User; u != nil {
1807 auth := &socksUsernamePassword{
1808 Username: u.Username(),
1809 }
1810 auth.Password, _ = u.Password()
1811 d.AuthMethods = []socksAuthMethod{
1812 socksAuthMethodNotRequired,
1813 socksAuthMethodUsernamePassword,
1814 }
1815 d.Authenticate = auth.Authenticate
1816 }
1817 if _, err := d.DialWithConn(ctx, conn, "tcp", cm.targetAddr); err != nil {
1818 conn.Close()
1819 return nil, err
1820 }
1821 case cm.targetScheme == "http":
1822 pconn.isProxy = true
1823 if pa := cm.proxyAuth(); pa != "" {
1824 pconn.mutateHeaderFunc = func(h Header) {
1825 h.Set("Proxy-Authorization", pa)
1826 }
1827 }
1828 case cm.targetScheme == "https":
1829 conn := pconn.conn
1830 var hdr Header
1831 if t.GetProxyConnectHeader != nil {
1832 var err error
1833 hdr, err = t.GetProxyConnectHeader(ctx, cm.proxyURL, cm.targetAddr)
1834 if err != nil {
1835 conn.Close()
1836 return nil, err
1837 }
1838 } else {
1839 hdr = t.ProxyConnectHeader
1840 }
1841 if hdr == nil {
1842 hdr = make(Header)
1843 }
1844 if pa := cm.proxyAuth(); pa != "" {
1845 hdr = hdr.Clone()
1846 hdr.Set("Proxy-Authorization", pa)
1847 }
1848 connectReq := &Request{
1849 Method: "CONNECT",
1850 URL: &url.URL{Opaque: cm.targetAddr},
1851 Host: cm.targetAddr,
1852 Header: hdr,
1853 }
1854 1855 // Set a (long) timeout here to make sure we don't block forever
1856 // and leak a goroutine if the connection stops replying after
1857 // the TCP connect.
1858 connectCtx, cancel := testHookProxyConnectTimeout(ctx, 1*time.Minute)
1859 defer cancel()
1860 1861 didReadResponse := chan struct{}{} // closed after CONNECT write+read is done or fails
1862 var (
1863 resp *Response
1864 err error // write or read error
1865 )
1866 // Write the CONNECT request & read the response.
1867 func() {
1868 defer close(didReadResponse)
1869 err = connectReq.Write(conn)
1870 if err != nil {
1871 return
1872 }
1873 // Okay to use and discard buffered reader here, because
1874 // TLS server will not speak until spoken to.
1875 br := bufio.NewReader(conn)
1876 resp, err = ReadResponse(br, connectReq)
1877 }()
1878 select {
1879 case <-connectCtx.Done():
1880 conn.Close()
1881 <-didReadResponse
1882 return nil, connectCtx.Err()
1883 case <-didReadResponse:
1884 // resp or err now set
1885 }
1886 if err != nil {
1887 conn.Close()
1888 return nil, err
1889 }
1890 1891 if t.OnProxyConnectResponse != nil {
1892 err = t.OnProxyConnectResponse(ctx, cm.proxyURL, connectReq, resp)
1893 if err != nil {
1894 conn.Close()
1895 return nil, err
1896 }
1897 }
1898 1899 if resp.StatusCode != 200 {
1900 _, text, ok := bytes.Cut(resp.Status, " ")
1901 conn.Close()
1902 if !ok {
1903 return nil, errors.New("unknown status code")
1904 }
1905 return nil, errors.New(text)
1906 }
1907 }
1908 1909 if cm.proxyURL != nil && cm.targetScheme == "https" {
1910 if err := pconn.addTLS(ctx, cm.tlsHost(), trace); err != nil {
1911 return nil, err
1912 }
1913 }
1914 1915 // Possible unencrypted HTTP/2 with prior knowledge.
1916 unencryptedHTTP2 := pconn.tlsState == nil &&
1917 t.Protocols != nil &&
1918 t.Protocols.UnencryptedHTTP2() &&
1919 !t.Protocols.HTTP1()
1920 if unencryptedHTTP2 {
1921 next, ok := t.TLSNextProto[nextProtoUnencryptedHTTP2]
1922 if !ok {
1923 return nil, errors.New("http: Transport does not support unencrypted HTTP/2")
1924 }
1925 alt := next(cm.targetAddr, unencryptedTLSConn(pconn.conn))
1926 if e, ok := alt.(erringRoundTripper); ok {
1927 // pconn.conn was closed by next (http2configureTransports.upgradeFn).
1928 return nil, e.RoundTripErr()
1929 }
1930 return &persistConn{t: t, cacheKey: pconn.cacheKey, alt: alt}, nil
1931 }
1932 1933 if s := pconn.tlsState; s != nil && s.NegotiatedProtocolIsMutual && s.NegotiatedProtocol != "" {
1934 if next, ok := t.TLSNextProto[s.NegotiatedProtocol]; ok {
1935 alt := next(cm.targetAddr, pconn.conn.(*tls.Conn))
1936 if e, ok := alt.(erringRoundTripper); ok {
1937 // pconn.conn was closed by next (http2configureTransports.upgradeFn).
1938 return nil, e.RoundTripErr()
1939 }
1940 return &persistConn{t: t, cacheKey: pconn.cacheKey, alt: alt}, nil
1941 }
1942 }
1943 1944 pconn.br = bufio.NewReaderSize(pconn, t.readBufferSize())
1945 pconn.bw = bufio.NewWriterSize(persistConnWriter{pconn}, t.writeBufferSize())
1946 1947 pconn.readLoop()
1948 pconn.writeLoop()
1949 return pconn, nil
1950 }
1951 1952 // persistConnWriter is the io.Writer written to by pc.bw.
1953 // It accumulates the number of bytes written to the underlying conn,
1954 // so the retry logic can determine whether any bytes made it across
1955 // the wire.
1956 // This is exactly 1 pointer field wide so it can go into an interface
1957 // without allocation.
1958 type persistConnWriter struct {
1959 pc *persistConn
1960 }
1961 1962 func (w persistConnWriter) Write(p []byte) (n int, err error) {
1963 n, err = w.pc.conn.Write(p)
1964 w.pc.nwrite += int64(n)
1965 return
1966 }
1967 1968 // ReadFrom exposes persistConnWriter's underlying Conn to io.Copy and if
1969 // the Conn implements io.ReaderFrom, it can take advantage of optimizations
1970 // such as sendfile.
1971 func (w persistConnWriter) ReadFrom(r io.Reader) (n int64, err error) {
1972 n, err = io.Copy(w.pc.conn, r)
1973 w.pc.nwrite += n
1974 return
1975 }
1976 1977 var _ io.ReaderFrom = (*persistConnWriter)(nil)
1978 1979 // connectMethod is the map key (in its String form) for keeping persistent
1980 // TCP connections alive for subsequent HTTP requests.
1981 //
1982 // A connect method may be of the following types:
1983 //
1984 // connectMethod.key().String() Description
1985 // ------------------------------ -------------------------
1986 // |http|foo.com http directly to server, no proxy
1987 // |https|foo.com https directly to server, no proxy
1988 // |https,h1|foo.com https directly to server w/o HTTP/2, no proxy
1989 // http://proxy.com|https|foo.com http to proxy, then CONNECT to foo.com
1990 // http://proxy.com|http http to proxy, http to anywhere after that
1991 // socks5://proxy.com|http|foo.com socks5 to proxy, then http to foo.com
1992 // socks5://proxy.com|https|foo.com socks5 to proxy, then https to foo.com
1993 // https://proxy.com|https|foo.com https to proxy, then CONNECT to foo.com
1994 // https://proxy.com|http https to proxy, http to anywhere after that
1995 type connectMethod struct {
1996 _ incomparable
1997 proxyURL *url.URL // nil for no proxy, else full proxy URL
1998 targetScheme string // "http" or "https"
1999 // If proxyURL specifies an http or https proxy, and targetScheme is http (not https),
2000 // then targetAddr is not included in the connect method key, because the socket can
2001 // be reused for different targetAddr values.
2002 targetAddr string
2003 onlyH1 bool // whether to disable HTTP/2 and force HTTP/1
2004 }
2005 2006 func (cm *connectMethod) key() connectMethodKey {
2007 proxyStr := ""
2008 targetAddr := cm.targetAddr
2009 if cm.proxyURL != nil {
2010 proxyStr = cm.proxyURL.String()
2011 if (cm.proxyURL.Scheme == "http" || cm.proxyURL.Scheme == "https") && cm.targetScheme == "http" {
2012 targetAddr = ""
2013 }
2014 }
2015 return connectMethodKey{
2016 proxy: proxyStr,
2017 scheme: cm.targetScheme,
2018 addr: targetAddr,
2019 onlyH1: cm.onlyH1,
2020 }
2021 }
2022 2023 // scheme returns the first hop scheme: http, https, or socks5
2024 func (cm *connectMethod) scheme() string {
2025 if cm.proxyURL != nil {
2026 return cm.proxyURL.Scheme
2027 }
2028 return cm.targetScheme
2029 }
2030 2031 // addr returns the first hop "host:port" to which we need to TCP connect.
2032 func (cm *connectMethod) addr() string {
2033 if cm.proxyURL != nil {
2034 return canonicalAddr(cm.proxyURL)
2035 }
2036 return cm.targetAddr
2037 }
2038 2039 // tlsHost returns the host name to match against the peer's
2040 // TLS certificate.
2041 func (cm *connectMethod) tlsHost() string {
2042 h := cm.targetAddr
2043 if hasPort(h) {
2044 h = h[:bytes.LastIndex(h, ":")]
2045 }
2046 return h
2047 }
2048 2049 // connectMethodKey is the map key version of connectMethod, with a
2050 // stringified proxy URL (or the empty string) instead of a pointer to
2051 // a URL.
2052 type connectMethodKey struct {
2053 proxy, scheme, addr string
2054 onlyH1 bool
2055 }
2056 2057 func (k connectMethodKey) String() string {
2058 // Only used by tests.
2059 var h1 string
2060 if k.onlyH1 {
2061 h1 = ",h1"
2062 }
2063 return fmt.Sprintf("%s|%s%s|%s", k.proxy, k.scheme, h1, k.addr)
2064 }
2065 2066 // persistConn wraps a connection, usually a persistent one
2067 // (but may be used for non-keep-alive requests as well)
2068 type persistConn struct {
2069 // alt optionally specifies the TLS NextProto RoundTripper.
2070 // This is used for HTTP/2 today and future protocols later.
2071 // If it's non-nil, the rest of the fields are unused.
2072 alt RoundTripper
2073 2074 t *Transport
2075 cacheKey connectMethodKey
2076 conn net.Conn
2077 tlsState *tls.ConnectionState
2078 br *bufio.Reader // from conn
2079 bw *bufio.Writer // to conn
2080 nwrite int64 // bytes written
2081 reqch chan requestAndChan // written by roundTrip; read by readLoop
2082 writech chan writeRequest // written by roundTrip; read by writeLoop
2083 closech chan struct{} // closed when conn closed
2084 isProxy bool
2085 sawEOF bool // whether we've seen EOF from conn; owned by readLoop
2086 readLimit int64 // bytes allowed to be read; owned by readLoop
2087 // writeErrCh passes the request write error (usually nil)
2088 // from the writeLoop goroutine to the readLoop which passes
2089 // it off to the res.Body reader, which then uses it to decide
2090 // whether or not a connection can be reused. Issue 7569.
2091 writeErrCh chan error
2092 2093 writeLoopDone chan struct{} // closed when write loop ends
2094 2095 // Both guarded by Transport.idleMu:
2096 idleAt time.Time // time it last become idle
2097 idleTimer *time.Timer // holding an AfterFunc to close it
2098 2099 mu sync.Mutex // guards following fields
2100 numExpectedResponses int
2101 closed error // set non-nil when conn is closed, before closech is closed
2102 canceledErr error // set non-nil if conn is canceled
2103 broken bool // an error has happened on this connection; marked broken so it's not reused.
2104 reused bool // whether conn has had successful request/response and is being reused.
2105 // mutateHeaderFunc is an optional func to modify extra
2106 // headers on each outbound request before it's written. (the
2107 // original Request given to RoundTrip is not modified)
2108 mutateHeaderFunc func(Header)
2109 }
2110 2111 func (pc *persistConn) maxHeaderResponseSize() int64 {
2112 if v := pc.t.MaxResponseHeaderBytes; v != 0 {
2113 return v
2114 }
2115 return 10 << 20 // conservative default; same as http2
2116 }
2117 2118 func (pc *persistConn) Read(p []byte) (n int, err error) {
2119 if pc.readLimit <= 0 {
2120 return 0, fmt.Errorf("read limit of %d bytes exhausted", pc.maxHeaderResponseSize())
2121 }
2122 if int64(len(p)) > pc.readLimit {
2123 p = p[:pc.readLimit]
2124 }
2125 n, err = pc.conn.Read(p)
2126 if err == io.EOF {
2127 pc.sawEOF = true
2128 }
2129 pc.readLimit -= int64(n)
2130 return
2131 }
2132 2133 // isBroken reports whether this connection is in a known broken state.
2134 func (pc *persistConn) isBroken() bool {
2135 pc.mu.Lock()
2136 b := pc.closed != nil
2137 pc.mu.Unlock()
2138 return b
2139 }
2140 2141 // canceled returns non-nil if the connection was closed due to
2142 // CancelRequest or due to context cancellation.
2143 func (pc *persistConn) canceled() error {
2144 pc.mu.Lock()
2145 defer pc.mu.Unlock()
2146 return pc.canceledErr
2147 }
2148 2149 // isReused reports whether this connection has been used before.
2150 func (pc *persistConn) isReused() bool {
2151 pc.mu.Lock()
2152 r := pc.reused
2153 pc.mu.Unlock()
2154 return r
2155 }
2156 2157 func (pc *persistConn) cancelRequest(err error) {
2158 pc.mu.Lock()
2159 defer pc.mu.Unlock()
2160 pc.canceledErr = err
2161 pc.closeLocked(errRequestCanceled)
2162 }
2163 2164 // closeConnIfStillIdle closes the connection if it's still sitting idle.
2165 // This is what's called by the persistConn's idleTimer, and is run in its
2166 // own goroutine.
2167 func (pc *persistConn) closeConnIfStillIdle() {
2168 t := pc.t
2169 t.idleMu.Lock()
2170 defer t.idleMu.Unlock()
2171 if _, ok := t.idleLRU.m[pc]; !ok {
2172 // Not idle.
2173 return
2174 }
2175 t.removeIdleConnLocked(pc)
2176 pc.close(errIdleConnTimeout)
2177 }
2178 2179 // mapRoundTripError returns the appropriate error value for
2180 // persistConn.roundTrip.
2181 //
2182 // The provided err is the first error that (*persistConn).roundTrip
2183 // happened to receive from its select statement.
2184 //
2185 // The startBytesWritten value should be the value of pc.nwrite before the roundTrip
2186 // started writing the request.
2187 func (pc *persistConn) mapRoundTripError(req *transportRequest, startBytesWritten int64, err error) error {
2188 if err == nil {
2189 return nil
2190 }
2191 2192 // Wait for the writeLoop goroutine to terminate to avoid data
2193 // races on callers who mutate the request on failure.
2194 //
2195 // When resc in pc.roundTrip and hence rc.ch receives a responseAndError
2196 // with a non-nil error it implies that the persistConn is either closed
2197 // or closing. Waiting on pc.writeLoopDone is hence safe as all callers
2198 // close closech which in turn ensures writeLoop returns.
2199 <-pc.writeLoopDone
2200 2201 // If the request was canceled, that's better than network
2202 // failures that were likely the result of tearing down the
2203 // connection.
2204 if cerr := pc.canceled(); cerr != nil {
2205 return cerr
2206 }
2207 2208 // See if an error was set explicitly.
2209 req.mu.Lock()
2210 reqErr := req.err
2211 req.mu.Unlock()
2212 if reqErr != nil {
2213 return reqErr
2214 }
2215 2216 if err == errServerClosedIdle {
2217 // Don't decorate
2218 return err
2219 }
2220 2221 if _, ok := err.(transportReadFromServerError); ok {
2222 if pc.nwrite == startBytesWritten {
2223 return nothingWrittenError{err}
2224 }
2225 // Don't decorate
2226 return err
2227 }
2228 if pc.isBroken() {
2229 if pc.nwrite == startBytesWritten {
2230 return nothingWrittenError{err}
2231 }
2232 return fmt.Errorf("net/http: HTTP/1.x transport connection broken: %w", err)
2233 }
2234 return err
2235 }
2236 2237 // errCallerOwnsConn is an internal sentinel error used when we hand
2238 // off a writable response.Body to the caller. We use this to prevent
2239 // closing a net.Conn that is now owned by the caller.
2240 var errCallerOwnsConn = errors.New("read loop ending; caller owns writable underlying conn")
2241 2242 func (pc *persistConn) readLoop() {
2243 closeErr := errReadLoopExiting // default value, if not changed below
2244 defer func() {
2245 pc.close(closeErr)
2246 pc.t.removeIdleConn(pc)
2247 }()
2248 2249 tryPutIdleConn := func(treq *transportRequest) bool {
2250 trace := treq.trace
2251 if err := pc.t.tryPutIdleConn(pc); err != nil {
2252 closeErr = err
2253 if trace != nil && trace.PutIdleConn != nil && err != errKeepAlivesDisabled {
2254 trace.PutIdleConn(err)
2255 }
2256 return false
2257 }
2258 if trace != nil && trace.PutIdleConn != nil {
2259 trace.PutIdleConn(nil)
2260 }
2261 return true
2262 }
2263 2264 // eofc is used to block caller goroutines reading from Response.Body
2265 // at EOF until this goroutines has (potentially) added the connection
2266 // back to the idle pool.
2267 eofc := chan struct{}{}
2268 defer close(eofc) // unblock reader on errors
2269 2270 // Read this once, before loop starts. (to avoid races in tests)
2271 testHookMu.Lock()
2272 testHookReadLoopBeforeNextRead := testHookReadLoopBeforeNextRead
2273 testHookMu.Unlock()
2274 2275 alive := true
2276 for alive {
2277 pc.readLimit = pc.maxHeaderResponseSize()
2278 _, err := pc.br.Peek(1)
2279 2280 pc.mu.Lock()
2281 if pc.numExpectedResponses == 0 {
2282 pc.readLoopPeekFailLocked(err)
2283 pc.mu.Unlock()
2284 return
2285 }
2286 pc.mu.Unlock()
2287 2288 rc := <-pc.reqch
2289 trace := rc.treq.trace
2290 2291 var resp *Response
2292 if err == nil {
2293 resp, err = pc.readResponse(rc, trace)
2294 } else {
2295 err = transportReadFromServerError{err}
2296 closeErr = err
2297 }
2298 2299 if err != nil {
2300 if pc.readLimit <= 0 {
2301 err = fmt.Errorf("net/http: server response headers exceeded %d bytes; aborted", pc.maxHeaderResponseSize())
2302 }
2303 2304 select {
2305 case rc.ch <- responseAndError{err: err}:
2306 case <-rc.callerGone:
2307 return
2308 }
2309 return
2310 }
2311 pc.readLimit = maxInt64 // effectively no limit for response bodies
2312 2313 pc.mu.Lock()
2314 pc.numExpectedResponses--
2315 pc.mu.Unlock()
2316 2317 bodyWritable := resp.bodyIsWritable()
2318 hasBody := rc.treq.Request.Method != "HEAD" && resp.ContentLength != 0
2319 2320 if resp.Close || rc.treq.Request.Close || resp.StatusCode <= 199 || bodyWritable {
2321 // Don't do keep-alive on error if either party requested a close
2322 // or we get an unexpected informational (1xx) response.
2323 // StatusCode 100 is already handled above.
2324 alive = false
2325 }
2326 2327 if !hasBody || bodyWritable {
2328 // Put the idle conn back into the pool before we send the response
2329 // so if they process it quickly and make another request, they'll
2330 // get this same conn. But we use the unbuffered channel 'rc'
2331 // to guarantee that persistConn.roundTrip got out of its select
2332 // potentially waiting for this persistConn to close.
2333 alive = alive &&
2334 !pc.sawEOF &&
2335 pc.wroteRequest() &&
2336 tryPutIdleConn(rc.treq)
2337 2338 if bodyWritable {
2339 closeErr = errCallerOwnsConn
2340 }
2341 2342 select {
2343 case rc.ch <- responseAndError{res: resp}:
2344 case <-rc.callerGone:
2345 return
2346 }
2347 2348 rc.treq.cancel(errRequestDone)
2349 2350 // Now that they've read from the unbuffered channel, they're safely
2351 // out of the select that also waits on this goroutine to die, so
2352 // we're allowed to exit now if needed (if alive is false)
2353 testHookReadLoopBeforeNextRead()
2354 continue
2355 }
2356 2357 waitForBodyRead := chan bool{2}
2358 body := &bodyEOFSignal{
2359 body: resp.Body,
2360 earlyCloseFn: func() error {
2361 waitForBodyRead <- false
2362 <-eofc // will be closed by deferred call at the end of the function
2363 return nil
2364 2365 },
2366 fn: func(err error) error {
2367 isEOF := err == io.EOF
2368 waitForBodyRead <- isEOF
2369 if isEOF {
2370 <-eofc // see comment above eofc declaration
2371 } else if err != nil {
2372 if cerr := pc.canceled(); cerr != nil {
2373 return cerr
2374 }
2375 }
2376 return err
2377 },
2378 }
2379 2380 resp.Body = body
2381 if rc.addedGzip && ascii.EqualFold(resp.Header.Get("Content-Encoding"), "gzip") {
2382 resp.Body = &gzipReader{body: body}
2383 resp.Header.Del("Content-Encoding")
2384 resp.Header.Del("Content-Length")
2385 resp.ContentLength = -1
2386 resp.Uncompressed = true
2387 }
2388 2389 select {
2390 case rc.ch <- responseAndError{res: resp}:
2391 case <-rc.callerGone:
2392 return
2393 }
2394 2395 // Before looping back to the top of this function and peeking on
2396 // the bufio.Reader, wait for the caller goroutine to finish
2397 // reading the response body. (or for cancellation or death)
2398 select {
2399 case bodyEOF := <-waitForBodyRead:
2400 alive = alive &&
2401 bodyEOF &&
2402 !pc.sawEOF &&
2403 pc.wroteRequest() &&
2404 tryPutIdleConn(rc.treq)
2405 if bodyEOF {
2406 eofc <- struct{}{}
2407 }
2408 case <-rc.treq.ctx.Done():
2409 alive = false
2410 pc.cancelRequest(context.Cause(rc.treq.ctx))
2411 case <-pc.closech:
2412 alive = false
2413 }
2414 2415 rc.treq.cancel(errRequestDone)
2416 testHookReadLoopBeforeNextRead()
2417 }
2418 }
2419 2420 func (pc *persistConn) readLoopPeekFailLocked(peekErr error) {
2421 if pc.closed != nil {
2422 return
2423 }
2424 if n := pc.br.Buffered(); n > 0 {
2425 buf, _ := pc.br.Peek(n)
2426 if is408Message(buf) {
2427 pc.closeLocked(errServerClosedIdle)
2428 return
2429 } else {
2430 log.Printf("Unsolicited response received on idle HTTP channel starting with %q; err=%v", buf, peekErr)
2431 }
2432 }
2433 if peekErr == io.EOF {
2434 // common case.
2435 pc.closeLocked(errServerClosedIdle)
2436 } else {
2437 pc.closeLocked(fmt.Errorf("readLoopPeekFailLocked: %w", peekErr))
2438 }
2439 }
2440 2441 // is408Message reports whether buf has the prefix of an
2442 // HTTP 408 Request Timeout response.
2443 // See golang.org/issue/32310.
2444 func is408Message(buf []byte) bool {
2445 if len(buf) < len("HTTP/1.x 408") {
2446 return false
2447 }
2448 if string(buf[:7]) != "HTTP/1." {
2449 return false
2450 }
2451 return string(buf[8:12]) == " 408"
2452 }
2453 2454 // readResponse reads an HTTP response (or two, in the case of "Expect:
2455 // 100-continue") from the server. It returns the final non-100 one.
2456 // trace is optional.
2457 func (pc *persistConn) readResponse(rc requestAndChan, trace *httptrace.ClientTrace) (resp *Response, err error) {
2458 if trace != nil && trace.GotFirstResponseByte != nil {
2459 if peek, err := pc.br.Peek(1); err == nil && len(peek) == 1 {
2460 trace.GotFirstResponseByte()
2461 }
2462 }
2463 2464 continueCh := rc.continueCh
2465 for {
2466 resp, err = ReadResponse(pc.br, rc.treq.Request)
2467 if err != nil {
2468 return
2469 }
2470 resCode := resp.StatusCode
2471 if continueCh != nil && resCode == StatusContinue {
2472 if trace != nil && trace.Got100Continue != nil {
2473 trace.Got100Continue()
2474 }
2475 continueCh <- struct{}{}
2476 continueCh = nil
2477 }
2478 is1xx := 100 <= resCode && resCode <= 199
2479 // treat 101 as a terminal status, see issue 26161
2480 is1xxNonTerminal := is1xx && resCode != StatusSwitchingProtocols
2481 if is1xxNonTerminal {
2482 if trace != nil && trace.Got1xxResponse != nil {
2483 if err := trace.Got1xxResponse(resCode, textproto.MIMEHeader(resp.Header)); err != nil {
2484 return nil, err
2485 }
2486 // If the 1xx response was delivered to the user,
2487 // then they're responsible for limiting the number of
2488 // responses. Reset the header limit.
2489 //
2490 // If the user didn't examine the 1xx response, then we
2491 // limit the size of all headers (including both 1xx
2492 // and the final response) to maxHeaderResponseSize.
2493 pc.readLimit = pc.maxHeaderResponseSize() // reset the limit
2494 }
2495 continue
2496 }
2497 break
2498 }
2499 if resp.isProtocolSwitch() {
2500 resp.Body = newReadWriteCloserBody(pc.br, pc.conn)
2501 }
2502 if continueCh != nil {
2503 // We send an "Expect: 100-continue" header, but the server
2504 // responded with a terminal status and no 100 Continue.
2505 //
2506 // If we're going to keep using the connection, we need to send the request body.
2507 // Tell writeLoop to skip sending the body if we're going to close the connection,
2508 // or to send it otherwise.
2509 //
2510 // The case where we receive a 101 Switching Protocols response is a bit
2511 // ambiguous, since we don't know what protocol we're switching to.
2512 // Conceivably, it's one that doesn't need us to send the body.
2513 // Given that we'll send the body if ExpectContinueTimeout expires,
2514 // be consistent and always send it if we aren't closing the connection.
2515 if resp.Close || rc.treq.Request.Close {
2516 close(continueCh) // don't send the body; the connection will close
2517 } else {
2518 continueCh <- struct{}{} // send the body
2519 }
2520 }
2521 2522 resp.TLS = pc.tlsState
2523 return
2524 }
2525 2526 // waitForContinue returns the function to block until
2527 // any response, timeout or connection close. After any of them,
2528 // the function returns a bool which indicates if the body should be sent.
2529 func (pc *persistConn) waitForContinue(continueCh <-chan struct{}) func() bool {
2530 if continueCh == nil {
2531 return nil
2532 }
2533 return func() bool {
2534 timer := time.NewTimer(pc.t.ExpectContinueTimeout)
2535 defer timer.Stop()
2536 2537 select {
2538 case _, ok := <-continueCh:
2539 return ok
2540 case <-timer.C:
2541 return true
2542 case <-pc.closech:
2543 return false
2544 }
2545 }
2546 }
2547 2548 func newReadWriteCloserBody(br *bufio.Reader, rwc io.ReadWriteCloser) io.ReadWriteCloser {
2549 body := &readWriteCloserBody{ReadWriteCloser: rwc}
2550 if br.Buffered() != 0 {
2551 body.br = br
2552 }
2553 return body
2554 }
2555 2556 // readWriteCloserBody is the Response.Body type used when we want to
2557 // give users write access to the Body through the underlying
2558 // connection (TCP, unless using custom dialers). This is then
2559 // the concrete type for a Response.Body on the 101 Switching
2560 // Protocols response, as used by WebSockets, h2c, etc.
2561 type readWriteCloserBody struct {
2562 _ incomparable
2563 br *bufio.Reader // used until empty
2564 io.ReadWriteCloser
2565 }
2566 2567 func (b *readWriteCloserBody) Read(p []byte) (n int, err error) {
2568 if b.br != nil {
2569 if n := b.br.Buffered(); len(p) > n {
2570 p = p[:n]
2571 }
2572 n, err = b.br.Read(p)
2573 if b.br.Buffered() == 0 {
2574 b.br = nil
2575 }
2576 return n, err
2577 }
2578 return b.ReadWriteCloser.Read(p)
2579 }
2580 2581 func (b *readWriteCloserBody) CloseWrite() error {
2582 if cw, ok := b.ReadWriteCloser.(interface{ CloseWrite() error }); ok {
2583 return cw.CloseWrite()
2584 }
2585 return fmt.Errorf("CloseWrite: %w", ErrNotSupported)
2586 }
2587 2588 // nothingWrittenError wraps a write errors which ended up writing zero bytes.
2589 type nothingWrittenError struct {
2590 error
2591 }
2592 2593 func (nwe nothingWrittenError) Unwrap() error {
2594 return nwe.error
2595 }
2596 2597 func (pc *persistConn) writeLoop() {
2598 defer close(pc.writeLoopDone)
2599 for {
2600 select {
2601 case wr := <-pc.writech:
2602 startBytesWritten := pc.nwrite
2603 err := wr.req.Request.write(pc.bw, pc.isProxy, wr.req.extra, pc.waitForContinue(wr.continueCh))
2604 if bre, ok := err.(requestBodyReadError); ok {
2605 err = bre.error
2606 // Errors reading from the user's
2607 // Request.Body are high priority.
2608 // Set it here before sending on the
2609 // channels below or calling
2610 // pc.close() which tears down
2611 // connections and causes other
2612 // errors.
2613 wr.req.setError(err)
2614 }
2615 if err == nil {
2616 err = pc.bw.Flush()
2617 }
2618 if err != nil {
2619 if pc.nwrite == startBytesWritten {
2620 err = nothingWrittenError{err}
2621 }
2622 }
2623 pc.writeErrCh <- err // to the body reader, which might recycle us
2624 wr.ch <- err // to the roundTrip function
2625 if err != nil {
2626 pc.close(err)
2627 return
2628 }
2629 case <-pc.closech:
2630 return
2631 }
2632 }
2633 }
2634 2635 // maxWriteWaitBeforeConnReuse is how long the a Transport RoundTrip
2636 // will wait to see the Request's Body.Write result after getting a
2637 // response from the server. See comments in (*persistConn).wroteRequest.
2638 //
2639 // In tests, we set this to a large value to avoid flakiness from inconsistent
2640 // recycling of connections.
2641 var maxWriteWaitBeforeConnReuse = 50 * time.Millisecond
2642 2643 // wroteRequest is a check before recycling a connection that the previous write
2644 // (from writeLoop above) happened and was successful.
2645 func (pc *persistConn) wroteRequest() bool {
2646 select {
2647 case err := <-pc.writeErrCh:
2648 // Common case: the write happened well before the response, so
2649 // avoid creating a timer.
2650 return err == nil
2651 default:
2652 // Rare case: the request was written in writeLoop above but
2653 // before it could send to pc.writeErrCh, the reader read it
2654 // all, processed it, and called us here. In this case, give the
2655 // write goroutine a bit of time to finish its send.
2656 //
2657 // Less rare case: We also get here in the legitimate case of
2658 // Issue 7569, where the writer is still writing (or stalled),
2659 // but the server has already replied. In this case, we don't
2660 // want to wait too long, and we want to return false so this
2661 // connection isn't re-used.
2662 t := time.NewTimer(maxWriteWaitBeforeConnReuse)
2663 defer t.Stop()
2664 select {
2665 case err := <-pc.writeErrCh:
2666 return err == nil
2667 case <-t.C:
2668 return false
2669 }
2670 }
2671 }
2672 2673 // responseAndError is how the goroutine reading from an HTTP/1 server
2674 // communicates with the goroutine doing the RoundTrip.
2675 type responseAndError struct {
2676 _ incomparable
2677 res *Response // else use this response (see res method)
2678 err error
2679 }
2680 2681 type requestAndChan struct {
2682 _ incomparable
2683 treq *transportRequest
2684 ch chan responseAndError // unbuffered; always send in select on callerGone
2685 2686 // whether the Transport (as opposed to the user client code)
2687 // added the Accept-Encoding gzip header. If the Transport
2688 // set it, only then do we transparently decode the gzip.
2689 addedGzip bool
2690 2691 // Optional blocking chan for Expect: 100-continue (for send).
2692 // If the request has an "Expect: 100-continue" header and
2693 // the server responds 100 Continue, readLoop send a value
2694 // to writeLoop via this chan.
2695 continueCh chan<- struct{}
2696 2697 callerGone <-chan struct{} // closed when roundTrip caller has returned
2698 }
2699 2700 // A writeRequest is sent by the caller's goroutine to the
2701 // writeLoop's goroutine to write a request while the read loop
2702 // concurrently waits on both the write response and the server's
2703 // reply.
2704 type writeRequest struct {
2705 req *transportRequest
2706 ch chan<- error
2707 2708 // Optional blocking chan for Expect: 100-continue (for receive).
2709 // If not nil, writeLoop blocks sending request body until
2710 // it receives from this chan.
2711 continueCh <-chan struct{}
2712 }
2713 2714 // httpTimeoutError represents a timeout.
2715 // It implements net.Error and wraps context.DeadlineExceeded.
2716 type timeoutError struct {
2717 err string
2718 }
2719 2720 func (e *timeoutError) Error() string { return e.err }
2721 func (e *timeoutError) Timeout() bool { return true }
2722 func (e *timeoutError) Temporary() bool { return true }
2723 func (e *timeoutError) Is(err error) bool { return err == context.DeadlineExceeded }
2724 2725 var errTimeout error = &timeoutError{"net/http: timeout awaiting response headers"}
2726 2727 // errRequestCanceled is set to be identical to the one from h2 to facilitate
2728 // testing.
2729 var errRequestCanceled = http2errRequestCanceled
2730 var errRequestCanceledConn = errors.New("net/http: request canceled while waiting for connection") // TODO: unify?
2731 2732 // errRequestDone is used to cancel the round trip Context after a request is successfully done.
2733 // It should not be seen by the user.
2734 var errRequestDone = errors.New("net/http: request completed")
2735 2736 func nop() {}
2737 2738 // testHooks. Always non-nil.
2739 var (
2740 testHookEnterRoundTrip = nop
2741 testHookWaitResLoop = nop
2742 testHookRoundTripRetried = nop
2743 testHookPrePendingDial = nop
2744 testHookPostPendingDial = nop
2745 2746 testHookMu sync.Locker = fakeLocker{} // guards following
2747 testHookReadLoopBeforeNextRead = nop
2748 )
2749 2750 func (pc *persistConn) roundTrip(req *transportRequest) (resp *Response, err error) {
2751 testHookEnterRoundTrip()
2752 pc.mu.Lock()
2753 pc.numExpectedResponses++
2754 headerFn := pc.mutateHeaderFunc
2755 pc.mu.Unlock()
2756 2757 if headerFn != nil {
2758 headerFn(req.extraHeaders())
2759 }
2760 2761 // Ask for a compressed version if the caller didn't set their
2762 // own value for Accept-Encoding. We only attempt to
2763 // uncompress the gzip stream if we were the layer that
2764 // requested it.
2765 requestedGzip := false
2766 if !pc.t.DisableCompression &&
2767 req.Header.Get("Accept-Encoding") == "" &&
2768 req.Header.Get("Range") == "" &&
2769 req.Method != "HEAD" {
2770 // Request gzip only, not deflate. Deflate is ambiguous and
2771 // not as universally supported anyway.
2772 // See: https://zlib.net/zlib_faq.html#faq39
2773 //
2774 // Note that we don't request this for HEAD requests,
2775 // due to a bug in nginx:
2776 // https://trac.nginx.org/nginx/ticket/358
2777 // https://golang.org/issue/5522
2778 //
2779 // We don't request gzip if the request is for a range, since
2780 // auto-decoding a portion of a gzipped document will just fail
2781 // anyway. See https://golang.org/issue/8923
2782 requestedGzip = true
2783 req.extraHeaders().Set("Accept-Encoding", "gzip")
2784 }
2785 2786 var continueCh chan struct{}
2787 if req.ProtoAtLeast(1, 1) && req.Body != nil && req.expectsContinue() {
2788 continueCh = chan struct{}{1}
2789 }
2790 2791 if pc.t.DisableKeepAlives &&
2792 !req.wantsClose() &&
2793 !isProtocolSwitchHeader(req.Header) {
2794 req.extraHeaders().Set("Connection", "close")
2795 }
2796 2797 gone := chan struct{}{}
2798 defer close(gone)
2799 2800 const debugRoundTrip = false
2801 2802 // Write the request concurrently with waiting for a response,
2803 // in case the server decides to reply before reading our full
2804 // request body.
2805 startBytesWritten := pc.nwrite
2806 writeErrCh := chan error{1}
2807 pc.writech <- writeRequest{req, writeErrCh, continueCh}
2808 2809 resc := chan responseAndError{}
2810 pc.reqch <- requestAndChan{
2811 treq: req,
2812 ch: resc,
2813 addedGzip: requestedGzip,
2814 continueCh: continueCh,
2815 callerGone: gone,
2816 }
2817 2818 handleResponse := func(re responseAndError) (*Response, error) {
2819 if (re.res == nil) == (re.err == nil) {
2820 panic(fmt.Sprintf("internal error: exactly one of res or err should be set; nil=%v", re.res == nil))
2821 }
2822 if debugRoundTrip {
2823 req.logf("resc recv: %p, %T/%#v", re.res, re.err, re.err)
2824 }
2825 if re.err != nil {
2826 return nil, pc.mapRoundTripError(req, startBytesWritten, re.err)
2827 }
2828 return re.res, nil
2829 }
2830 2831 var respHeaderTimer <-chan time.Time
2832 ctxDoneChan := req.ctx.Done()
2833 pcClosed := pc.closech
2834 for {
2835 testHookWaitResLoop()
2836 select {
2837 case err := <-writeErrCh:
2838 if debugRoundTrip {
2839 req.logf("writeErrCh recv: %T/%#v", err, err)
2840 }
2841 if err != nil {
2842 pc.close(fmt.Errorf("write error: %w", err))
2843 return nil, pc.mapRoundTripError(req, startBytesWritten, err)
2844 }
2845 if d := pc.t.ResponseHeaderTimeout; d > 0 {
2846 if debugRoundTrip {
2847 req.logf("starting timer for %v", d)
2848 }
2849 timer := time.NewTimer(d)
2850 defer timer.Stop() // prevent leaks
2851 respHeaderTimer = timer.C
2852 }
2853 case <-pcClosed:
2854 select {
2855 case re := <-resc:
2856 // The pconn closing raced with the response to the request,
2857 // probably after the server wrote a response and immediately
2858 // closed the connection. Use the response.
2859 return handleResponse(re)
2860 default:
2861 }
2862 if debugRoundTrip {
2863 req.logf("closech recv: %T %#v", pc.closed, pc.closed)
2864 }
2865 return nil, pc.mapRoundTripError(req, startBytesWritten, pc.closed)
2866 case <-respHeaderTimer:
2867 if debugRoundTrip {
2868 req.logf("timeout waiting for response headers.")
2869 }
2870 pc.close(errTimeout)
2871 return nil, errTimeout
2872 case re := <-resc:
2873 return handleResponse(re)
2874 case <-ctxDoneChan:
2875 select {
2876 case re := <-resc:
2877 // readLoop is responsible for canceling req.ctx after
2878 // it reads the response body. Check for a response racing
2879 // the context close, and use the response if available.
2880 return handleResponse(re)
2881 default:
2882 }
2883 pc.cancelRequest(context.Cause(req.ctx))
2884 }
2885 }
2886 }
2887 2888 // tLogKey is a context WithValue key for test debugging contexts containing
2889 // a t.Logf func. See export_test.go's Request.WithT method.
2890 type tLogKey struct{}
2891 2892 func (tr *transportRequest) logf(format string, args ...any) {
2893 if logf, ok := tr.Request.Context().Value(tLogKey{}).(func(string, ...any)); ok {
2894 logf(time.Now().Format(time.RFC3339Nano)+": "+format, args...)
2895 }
2896 }
2897 2898 // markReused marks this connection as having been successfully used for a
2899 // request and response.
2900 func (pc *persistConn) markReused() {
2901 pc.mu.Lock()
2902 pc.reused = true
2903 pc.mu.Unlock()
2904 }
2905 2906 // close closes the underlying TCP connection and closes
2907 // the pc.closech channel.
2908 //
2909 // The provided err is only for testing and debugging; in normal
2910 // circumstances it should never be seen by users.
2911 func (pc *persistConn) close(err error) {
2912 pc.mu.Lock()
2913 defer pc.mu.Unlock()
2914 pc.closeLocked(err)
2915 }
2916 2917 func (pc *persistConn) closeLocked(err error) {
2918 if err == nil {
2919 panic("nil error")
2920 }
2921 pc.broken = true
2922 if pc.closed == nil {
2923 pc.closed = err
2924 pc.t.decConnsPerHost(pc.cacheKey)
2925 // Close HTTP/1 (pc.alt == nil) connection.
2926 // HTTP/2 closes its connection itself.
2927 if pc.alt == nil {
2928 if err != errCallerOwnsConn {
2929 pc.conn.Close()
2930 }
2931 close(pc.closech)
2932 }
2933 }
2934 pc.mutateHeaderFunc = nil
2935 }
2936 2937 func schemePort(scheme string) string {
2938 switch scheme {
2939 case "http":
2940 return "80"
2941 case "https":
2942 return "443"
2943 case "socks5", "socks5h":
2944 return "1080"
2945 default:
2946 return ""
2947 }
2948 }
2949 2950 func idnaASCIIFromURL(url *url.URL) string {
2951 addr := url.Hostname()
2952 if v, err := idnaASCII(addr); err == nil {
2953 addr = v
2954 }
2955 return addr
2956 }
2957 2958 // canonicalAddr returns url.Host but always with a ":port" suffix.
2959 func canonicalAddr(url *url.URL) string {
2960 port := url.Port()
2961 if port == "" {
2962 port = schemePort(url.Scheme)
2963 }
2964 return net.JoinHostPort(idnaASCIIFromURL(url), port)
2965 }
2966 2967 // bodyEOFSignal is used by the HTTP/1 transport when reading response
2968 // bodies to make sure we see the end of a response body before
2969 // proceeding and reading on the connection again.
2970 //
2971 // It wraps a ReadCloser but runs fn (if non-nil) at most
2972 // once, right before its final (error-producing) Read or Close call
2973 // returns. fn should return the new error to return from Read or Close.
2974 //
2975 // If earlyCloseFn is non-nil and Close is called before io.EOF is
2976 // seen, earlyCloseFn is called instead of fn, and its return value is
2977 // the return value from Close.
2978 type bodyEOFSignal struct {
2979 body io.ReadCloser
2980 mu sync.Mutex // guards following 4 fields
2981 closed bool // whether Close has been called
2982 rerr error // sticky Read error
2983 fn func(error) error // err will be nil on Read io.EOF
2984 earlyCloseFn func() error // optional alt Close func used if io.EOF not seen
2985 }
2986 2987 var errReadOnClosedResBody = errors.New("http: read on closed response body")
2988 2989 func (es *bodyEOFSignal) Read(p []byte) (n int, err error) {
2990 es.mu.Lock()
2991 closed, rerr := es.closed, es.rerr
2992 es.mu.Unlock()
2993 if closed {
2994 return 0, errReadOnClosedResBody
2995 }
2996 if rerr != nil {
2997 return 0, rerr
2998 }
2999 3000 n, err = es.body.Read(p)
3001 if err != nil {
3002 es.mu.Lock()
3003 defer es.mu.Unlock()
3004 if es.rerr == nil {
3005 es.rerr = err
3006 }
3007 err = es.condfn(err)
3008 }
3009 return
3010 }
3011 3012 func (es *bodyEOFSignal) Close() error {
3013 es.mu.Lock()
3014 defer es.mu.Unlock()
3015 if es.closed {
3016 return nil
3017 }
3018 es.closed = true
3019 if es.earlyCloseFn != nil && es.rerr != io.EOF {
3020 return es.earlyCloseFn()
3021 }
3022 err := es.body.Close()
3023 return es.condfn(err)
3024 }
3025 3026 // caller must hold es.mu.
3027 func (es *bodyEOFSignal) condfn(err error) error {
3028 if es.fn == nil {
3029 return err
3030 }
3031 err = es.fn(err)
3032 es.fn = nil
3033 return err
3034 }
3035 3036 // gzipReader wraps a response body so it can lazily
3037 // call gzip.NewReader on the first call to Read
3038 type gzipReader struct {
3039 _ incomparable
3040 body *bodyEOFSignal // underlying HTTP/1 response body framing
3041 zr *gzip.Reader // lazily-initialized gzip reader
3042 zerr error // any error from gzip.NewReader; sticky
3043 }
3044 3045 func (gz *gzipReader) Read(p []byte) (n int, err error) {
3046 if gz.zr == nil {
3047 if gz.zerr == nil {
3048 gz.zr, gz.zerr = gzip.NewReader(gz.body)
3049 }
3050 if gz.zerr != nil {
3051 return 0, gz.zerr
3052 }
3053 }
3054 3055 gz.body.mu.Lock()
3056 if gz.body.closed {
3057 err = errReadOnClosedResBody
3058 }
3059 gz.body.mu.Unlock()
3060 3061 if err != nil {
3062 return 0, err
3063 }
3064 return gz.zr.Read(p)
3065 }
3066 3067 func (gz *gzipReader) Close() error {
3068 return gz.body.Close()
3069 }
3070 3071 type tlsHandshakeTimeoutError struct{}
3072 3073 func (tlsHandshakeTimeoutError) Timeout() bool { return true }
3074 func (tlsHandshakeTimeoutError) Temporary() bool { return true }
3075 func (tlsHandshakeTimeoutError) Error() string { return "net/http: TLS handshake timeout" }
3076 3077 // fakeLocker is a sync.Locker which does nothing. It's used to guard
3078 // test-only fields when not under test, to avoid runtime atomic
3079 // overhead.
3080 type fakeLocker struct{}
3081 3082 func (fakeLocker) Lock() {}
3083 func (fakeLocker) Unlock() {}
3084 3085 // cloneTLSConfig returns a shallow clone of cfg, or a new zero tls.Config if
3086 // cfg is nil. This is safe to call even if cfg is in active use by a TLS
3087 // client or server.
3088 //
3089 // cloneTLSConfig should be an internal detail,
3090 // but widely used packages access it using linkname.
3091 // Notable members of the hall of shame include:
3092 // - github.com/searKing/golang
3093 //
3094 // Do not remove or change the type signature.
3095 // See go.dev/issue/67401.
3096 //
3097 //go:linkname cloneTLSConfig
3098 func cloneTLSConfig(cfg *tls.Config) *tls.Config {
3099 if cfg == nil {
3100 return &tls.Config{}
3101 }
3102 return cfg.Clone()
3103 }
3104 3105 type connLRU struct {
3106 ll *list.List // list.Element.Value type is of *persistConn
3107 m map[*persistConn]*list.Element
3108 }
3109 3110 // add adds pc to the head of the linked list.
3111 func (cl *connLRU) add(pc *persistConn) {
3112 if cl.ll == nil {
3113 cl.ll = list.New()
3114 cl.m = map[*persistConn]*list.Element{}
3115 }
3116 ele := cl.ll.PushFront(pc)
3117 if _, ok := cl.m[pc]; ok {
3118 panic("persistConn was already in LRU")
3119 }
3120 cl.m[pc] = ele
3121 }
3122 3123 func (cl *connLRU) removeOldest() *persistConn {
3124 ele := cl.ll.Back()
3125 pc := ele.Value.(*persistConn)
3126 cl.ll.Remove(ele)
3127 delete(cl.m, pc)
3128 return pc
3129 }
3130 3131 // remove removes pc from cl.
3132 func (cl *connLRU) remove(pc *persistConn) {
3133 if ele, ok := cl.m[pc]; ok {
3134 cl.ll.Remove(ele)
3135 delete(cl.m, pc)
3136 }
3137 }
3138 3139 // len returns the number of items in the cache.
3140 func (cl *connLRU) len() int {
3141 return len(cl.m)
3142 }
3143