diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index 4530ffdbc2e..503a360a3ab 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -1927,43 +1927,43 @@ }, { "ImportPath": "golang.org/x/net/context", - "Rev": "c2528b2dd8352441850638a8bb678c2ad056fd3e" + "Rev": "62685c2d7ca23c807425dca88b11a3e2323dab41" }, { "ImportPath": "golang.org/x/net/context/ctxhttp", - "Rev": "c2528b2dd8352441850638a8bb678c2ad056fd3e" + "Rev": "62685c2d7ca23c807425dca88b11a3e2323dab41" }, { "ImportPath": "golang.org/x/net/html", - "Rev": "c2528b2dd8352441850638a8bb678c2ad056fd3e" + "Rev": "62685c2d7ca23c807425dca88b11a3e2323dab41" }, { "ImportPath": "golang.org/x/net/html/atom", - "Rev": "c2528b2dd8352441850638a8bb678c2ad056fd3e" + "Rev": "62685c2d7ca23c807425dca88b11a3e2323dab41" }, { "ImportPath": "golang.org/x/net/http2", - "Rev": "c2528b2dd8352441850638a8bb678c2ad056fd3e" + "Rev": "62685c2d7ca23c807425dca88b11a3e2323dab41" }, { "ImportPath": "golang.org/x/net/http2/hpack", - "Rev": "c2528b2dd8352441850638a8bb678c2ad056fd3e" + "Rev": "62685c2d7ca23c807425dca88b11a3e2323dab41" }, { "ImportPath": "golang.org/x/net/internal/timeseries", - "Rev": "c2528b2dd8352441850638a8bb678c2ad056fd3e" + "Rev": "62685c2d7ca23c807425dca88b11a3e2323dab41" }, { "ImportPath": "golang.org/x/net/proxy", - "Rev": "c2528b2dd8352441850638a8bb678c2ad056fd3e" + "Rev": "62685c2d7ca23c807425dca88b11a3e2323dab41" }, { "ImportPath": "golang.org/x/net/trace", - "Rev": "c2528b2dd8352441850638a8bb678c2ad056fd3e" + "Rev": "62685c2d7ca23c807425dca88b11a3e2323dab41" }, { "ImportPath": "golang.org/x/net/websocket", - "Rev": "c2528b2dd8352441850638a8bb678c2ad056fd3e" + "Rev": "62685c2d7ca23c807425dca88b11a3e2323dab41" }, { "ImportPath": "golang.org/x/oauth2", diff --git a/vendor/golang.org/x/net/context/context.go b/vendor/golang.org/x/net/context/context.go index ef2f3e86fec..11bd8d34e6c 100644 --- a/vendor/golang.org/x/net/context/context.go +++ b/vendor/golang.org/x/net/context/context.go @@ -189,7 +189,7 @@ func Background() Context { } // TODO returns a non-nil, empty Context. Code should use context.TODO when -// it's unclear which Context to use or it's is not yet available (because the +// it's unclear which Context to use or it is not yet available (because the // surrounding function has not yet been extended to accept a Context // parameter). TODO is recognized by static analysis tools that determine // whether Contexts are propagated correctly in a program. diff --git a/vendor/golang.org/x/net/context/ctxhttp/cancelreq.go b/vendor/golang.org/x/net/context/ctxhttp/cancelreq.go index 48610e36277..e3170e3333a 100644 --- a/vendor/golang.org/x/net/context/ctxhttp/cancelreq.go +++ b/vendor/golang.org/x/net/context/ctxhttp/cancelreq.go @@ -9,6 +9,7 @@ package ctxhttp import "net/http" func canceler(client *http.Client, req *http.Request) func() { + // TODO(djd): Respect any existing value of req.Cancel. ch := make(chan struct{}) req.Cancel = ch diff --git a/vendor/golang.org/x/net/context/ctxhttp/ctxhttp.go b/vendor/golang.org/x/net/context/ctxhttp/ctxhttp.go index 9f34888132e..26a5e19ac3d 100644 --- a/vendor/golang.org/x/net/context/ctxhttp/ctxhttp.go +++ b/vendor/golang.org/x/net/context/ctxhttp/ctxhttp.go @@ -14,6 +14,14 @@ import ( "golang.org/x/net/context" ) +func nop() {} + +var ( + testHookContextDoneBeforeHeaders = nop + testHookDoReturned = nop + testHookDidBodyClose = nop +) + // Do sends an HTTP request with the provided http.Client and returns an HTTP response. // If the client is nil, http.DefaultClient is used. // If the context is canceled or times out, ctx.Err() will be returned. @@ -33,16 +41,44 @@ func Do(ctx context.Context, client *http.Client, req *http.Request) (*http.Resp go func() { resp, err := client.Do(req) + testHookDoReturned() result <- responseAndError{resp, err} }() + var resp *http.Response + select { case <-ctx.Done(): + testHookContextDoneBeforeHeaders() cancel() + // Clean up after the goroutine calling client.Do: + go func() { + if r := <-result; r.resp != nil { + testHookDidBodyClose() + r.resp.Body.Close() + } + }() return nil, ctx.Err() case r := <-result: - return r.resp, r.err + var err error + resp, err = r.resp, r.err + if err != nil { + return resp, err + } } + + c := make(chan struct{}) + go func() { + select { + case <-ctx.Done(): + cancel() + case <-c: + // The response's Body is closed. + } + }() + resp.Body = ¬ifyingReader{resp.Body, c} + + return resp, nil } // Get issues a GET request via the Do function. @@ -77,3 +113,28 @@ func Post(ctx context.Context, client *http.Client, url string, bodyType string, func PostForm(ctx context.Context, client *http.Client, url string, data url.Values) (*http.Response, error) { return Post(ctx, client, url, "application/x-www-form-urlencoded", strings.NewReader(data.Encode())) } + +// notifyingReader is an io.ReadCloser that closes the notify channel after +// Close is called or a Read fails on the underlying ReadCloser. +type notifyingReader struct { + io.ReadCloser + notify chan<- struct{} +} + +func (r *notifyingReader) Read(p []byte) (int, error) { + n, err := r.ReadCloser.Read(p) + if err != nil && r.notify != nil { + close(r.notify) + r.notify = nil + } + return n, err +} + +func (r *notifyingReader) Close() error { + err := r.ReadCloser.Close() + if r.notify != nil { + close(r.notify) + r.notify = nil + } + return err +} diff --git a/vendor/golang.org/x/net/http2/Dockerfile b/vendor/golang.org/x/net/http2/Dockerfile index b4e14d55a51..53fc5257974 100644 --- a/vendor/golang.org/x/net/http2/Dockerfile +++ b/vendor/golang.org/x/net/http2/Dockerfile @@ -17,8 +17,15 @@ RUN apt-get install -y --no-install-recommends \ libcunit1-dev libssl-dev libxml2-dev libevent-dev \ automake autoconf +# The list of packages nghttp2 recommends for h2load: +RUN apt-get install -y --no-install-recommends make binutils \ + autoconf automake autotools-dev \ + libtool pkg-config zlib1g-dev libcunit1-dev libssl-dev libxml2-dev \ + libev-dev libevent-dev libjansson-dev libjemalloc-dev \ + cython python3.4-dev python-setuptools + # Note: setting NGHTTP2_VER before the git clone, so an old git clone isn't cached: -ENV NGHTTP2_VER af24f8394e43f4 +ENV NGHTTP2_VER 895da9a RUN cd /root && git clone https://github.com/tatsuhiro-t/nghttp2.git WORKDIR /root/nghttp2 @@ -31,9 +38,9 @@ RUN make RUN make install WORKDIR /root -RUN wget http://curl.haxx.se/download/curl-7.40.0.tar.gz -RUN tar -zxvf curl-7.40.0.tar.gz -WORKDIR /root/curl-7.40.0 +RUN wget http://curl.haxx.se/download/curl-7.45.0.tar.gz +RUN tar -zxvf curl-7.45.0.tar.gz +WORKDIR /root/curl-7.45.0 RUN ./configure --with-ssl --with-nghttp2=/usr/local RUN make RUN make install diff --git a/vendor/golang.org/x/net/http2/README b/vendor/golang.org/x/net/http2/README index aba8ae5ef66..360d5aa3790 100644 --- a/vendor/golang.org/x/net/http2/README +++ b/vendor/golang.org/x/net/http2/README @@ -17,4 +17,4 @@ Demo test server at https://http2.golang.org/ Help & bug reports welcome! Contributing: https://golang.org/doc/contribute.html -Bugs: https://github.com/golang/go/issues/new?title=x/net/http2:+ +Bugs: https://golang.org/issue/new?title=x/net/http2:+ diff --git a/vendor/golang.org/x/net/http2/buffer.go b/vendor/golang.org/x/net/http2/buffer.go deleted file mode 100644 index c43954cf04d..00000000000 --- a/vendor/golang.org/x/net/http2/buffer.go +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright 2014 The Go Authors. -// See https://code.google.com/p/go/source/browse/CONTRIBUTORS -// Licensed under the same terms as Go itself: -// https://code.google.com/p/go/source/browse/LICENSE - -package http2 - -import ( - "errors" -) - -// buffer is an io.ReadWriteCloser backed by a fixed size buffer. -// It never allocates, but moves old data as new data is written. -type buffer struct { - buf []byte - r, w int - closed bool - err error // err to return to reader -} - -var ( - errReadEmpty = errors.New("read from empty buffer") - errWriteClosed = errors.New("write on closed buffer") - errWriteFull = errors.New("write on full buffer") -) - -// Read copies bytes from the buffer into p. -// It is an error to read when no data is available. -func (b *buffer) Read(p []byte) (n int, err error) { - n = copy(p, b.buf[b.r:b.w]) - b.r += n - if b.closed && b.r == b.w { - err = b.err - } else if b.r == b.w && n == 0 { - err = errReadEmpty - } - return n, err -} - -// Len returns the number of bytes of the unread portion of the buffer. -func (b *buffer) Len() int { - return b.w - b.r -} - -// Write copies bytes from p into the buffer. -// It is an error to write more data than the buffer can hold. -func (b *buffer) Write(p []byte) (n int, err error) { - if b.closed { - return 0, errWriteClosed - } - - // Slide existing data to beginning. - if b.r > 0 && len(p) > len(b.buf)-b.w { - copy(b.buf, b.buf[b.r:b.w]) - b.w -= b.r - b.r = 0 - } - - // Write new data. - n = copy(b.buf[b.w:], p) - b.w += n - if n < len(p) { - err = errWriteFull - } - return n, err -} - -// Close marks the buffer as closed. Future calls to Write will -// return an error. Future calls to Read, once the buffer is -// empty, will return err. -func (b *buffer) Close(err error) { - if !b.closed { - b.closed = true - b.err = err - } -} diff --git a/vendor/golang.org/x/net/http2/client_conn_pool.go b/vendor/golang.org/x/net/http2/client_conn_pool.go new file mode 100644 index 00000000000..772ea5e9244 --- /dev/null +++ b/vendor/golang.org/x/net/http2/client_conn_pool.go @@ -0,0 +1,225 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Transport code's client connection pooling. + +package http2 + +import ( + "crypto/tls" + "net/http" + "sync" +) + +// ClientConnPool manages a pool of HTTP/2 client connections. +type ClientConnPool interface { + GetClientConn(req *http.Request, addr string) (*ClientConn, error) + MarkDead(*ClientConn) +} + +// TODO: use singleflight for dialing and addConnCalls? +type clientConnPool struct { + t *Transport + + mu sync.Mutex // TODO: maybe switch to RWMutex + // TODO: add support for sharing conns based on cert names + // (e.g. share conn for googleapis.com and appspot.com) + conns map[string][]*ClientConn // key is host:port + dialing map[string]*dialCall // currently in-flight dials + keys map[*ClientConn][]string + addConnCalls map[string]*addConnCall // in-flight addConnIfNeede calls +} + +func (p *clientConnPool) GetClientConn(req *http.Request, addr string) (*ClientConn, error) { + return p.getClientConn(req, addr, dialOnMiss) +} + +const ( + dialOnMiss = true + noDialOnMiss = false +) + +func (p *clientConnPool) getClientConn(_ *http.Request, addr string, dialOnMiss bool) (*ClientConn, error) { + p.mu.Lock() + for _, cc := range p.conns[addr] { + if cc.CanTakeNewRequest() { + p.mu.Unlock() + return cc, nil + } + } + if !dialOnMiss { + p.mu.Unlock() + return nil, ErrNoCachedConn + } + call := p.getStartDialLocked(addr) + p.mu.Unlock() + <-call.done + return call.res, call.err +} + +// dialCall is an in-flight Transport dial call to a host. +type dialCall struct { + p *clientConnPool + done chan struct{} // closed when done + res *ClientConn // valid after done is closed + err error // valid after done is closed +} + +// requires p.mu is held. +func (p *clientConnPool) getStartDialLocked(addr string) *dialCall { + if call, ok := p.dialing[addr]; ok { + // A dial is already in-flight. Don't start another. + return call + } + call := &dialCall{p: p, done: make(chan struct{})} + if p.dialing == nil { + p.dialing = make(map[string]*dialCall) + } + p.dialing[addr] = call + go call.dial(addr) + return call +} + +// run in its own goroutine. +func (c *dialCall) dial(addr string) { + c.res, c.err = c.p.t.dialClientConn(addr) + close(c.done) + + c.p.mu.Lock() + delete(c.p.dialing, addr) + if c.err == nil { + c.p.addConnLocked(addr, c.res) + } + c.p.mu.Unlock() +} + +// addConnIfNeeded makes a NewClientConn out of c if a connection for key doesn't +// already exist. It coalesces concurrent calls with the same key. +// This is used by the http1 Transport code when it creates a new connection. Because +// the http1 Transport doesn't de-dup TCP dials to outbound hosts (because it doesn't know +// the protocol), it can get into a situation where it has multiple TLS connections. +// This code decides which ones live or die. +// The return value used is whether c was used. +// c is never closed. +func (p *clientConnPool) addConnIfNeeded(key string, t *Transport, c *tls.Conn) (used bool, err error) { + p.mu.Lock() + for _, cc := range p.conns[key] { + if cc.CanTakeNewRequest() { + p.mu.Unlock() + return false, nil + } + } + call, dup := p.addConnCalls[key] + if !dup { + if p.addConnCalls == nil { + p.addConnCalls = make(map[string]*addConnCall) + } + call = &addConnCall{ + p: p, + done: make(chan struct{}), + } + p.addConnCalls[key] = call + go call.run(t, key, c) + } + p.mu.Unlock() + + <-call.done + if call.err != nil { + return false, call.err + } + return !dup, nil +} + +type addConnCall struct { + p *clientConnPool + done chan struct{} // closed when done + err error +} + +func (c *addConnCall) run(t *Transport, key string, tc *tls.Conn) { + cc, err := t.NewClientConn(tc) + + p := c.p + p.mu.Lock() + if err != nil { + c.err = err + } else { + p.addConnLocked(key, cc) + } + delete(p.addConnCalls, key) + p.mu.Unlock() + close(c.done) +} + +func (p *clientConnPool) addConn(key string, cc *ClientConn) { + p.mu.Lock() + p.addConnLocked(key, cc) + p.mu.Unlock() +} + +// p.mu must be held +func (p *clientConnPool) addConnLocked(key string, cc *ClientConn) { + for _, v := range p.conns[key] { + if v == cc { + return + } + } + if p.conns == nil { + p.conns = make(map[string][]*ClientConn) + } + if p.keys == nil { + p.keys = make(map[*ClientConn][]string) + } + p.conns[key] = append(p.conns[key], cc) + p.keys[cc] = append(p.keys[cc], key) +} + +func (p *clientConnPool) MarkDead(cc *ClientConn) { + p.mu.Lock() + defer p.mu.Unlock() + for _, key := range p.keys[cc] { + vv, ok := p.conns[key] + if !ok { + continue + } + newList := filterOutClientConn(vv, cc) + if len(newList) > 0 { + p.conns[key] = newList + } else { + delete(p.conns, key) + } + } + delete(p.keys, cc) +} + +func (p *clientConnPool) closeIdleConnections() { + p.mu.Lock() + defer p.mu.Unlock() + // TODO: don't close a cc if it was just added to the pool + // milliseconds ago and has never been used. There's currently + // a small race window with the HTTP/1 Transport's integration + // where it can add an idle conn just before using it, and + // somebody else can concurrently call CloseIdleConns and + // break some caller's RoundTrip. + for _, vv := range p.conns { + for _, cc := range vv { + cc.closeIfIdle() + } + } +} + +func filterOutClientConn(in []*ClientConn, exclude *ClientConn) []*ClientConn { + out := in[:0] + for _, v := range in { + if v != exclude { + out = append(out, v) + } + } + // If we filtered it out, zero out the last item to prevent + // the GC from seeing it. + if len(in) != len(out) { + in[len(in)-1] = nil + } + return out +} diff --git a/vendor/golang.org/x/net/http2/configure_transport.go b/vendor/golang.org/x/net/http2/configure_transport.go new file mode 100644 index 00000000000..daa17f5d43b --- /dev/null +++ b/vendor/golang.org/x/net/http2/configure_transport.go @@ -0,0 +1,89 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build go1.6 + +package http2 + +import ( + "crypto/tls" + "fmt" + "net/http" +) + +func configureTransport(t1 *http.Transport) (*Transport, error) { + connPool := new(clientConnPool) + t2 := &Transport{ + ConnPool: noDialClientConnPool{connPool}, + t1: t1, + } + connPool.t = t2 + if err := registerHTTPSProtocol(t1, noDialH2RoundTripper{t2}); err != nil { + return nil, err + } + if t1.TLSClientConfig == nil { + t1.TLSClientConfig = new(tls.Config) + } + if !strSliceContains(t1.TLSClientConfig.NextProtos, "h2") { + t1.TLSClientConfig.NextProtos = append([]string{"h2"}, t1.TLSClientConfig.NextProtos...) + } + if !strSliceContains(t1.TLSClientConfig.NextProtos, "http/1.1") { + t1.TLSClientConfig.NextProtos = append(t1.TLSClientConfig.NextProtos, "http/1.1") + } + upgradeFn := func(authority string, c *tls.Conn) http.RoundTripper { + addr := authorityAddr(authority) + if used, err := connPool.addConnIfNeeded(addr, t2, c); err != nil { + go c.Close() + return erringRoundTripper{err} + } else if !used { + // Turns out we don't need this c. + // For example, two goroutines made requests to the same host + // at the same time, both kicking off TCP dials. (since protocol + // was unknown) + go c.Close() + } + return t2 + } + if m := t1.TLSNextProto; len(m) == 0 { + t1.TLSNextProto = map[string]func(string, *tls.Conn) http.RoundTripper{ + "h2": upgradeFn, + } + } else { + m["h2"] = upgradeFn + } + return t2, nil +} + +// registerHTTPSProtocol calls Transport.RegisterProtocol but +// convering panics into errors. +func registerHTTPSProtocol(t *http.Transport, rt http.RoundTripper) (err error) { + defer func() { + if e := recover(); e != nil { + err = fmt.Errorf("%v", e) + } + }() + t.RegisterProtocol("https", rt) + return nil +} + +// noDialClientConnPool is an implementation of http2.ClientConnPool +// which never dials. We let the HTTP/1.1 client dial and use its TLS +// connection instead. +type noDialClientConnPool struct{ *clientConnPool } + +func (p noDialClientConnPool) GetClientConn(req *http.Request, addr string) (*ClientConn, error) { + return p.getClientConn(req, addr, noDialOnMiss) +} + +// noDialH2RoundTripper is a RoundTripper which only tries to complete the request +// if there's already has a cached connection to the host. +type noDialH2RoundTripper struct{ t *Transport } + +func (rt noDialH2RoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { + res, err := rt.t.RoundTrip(req) + if err == ErrNoCachedConn { + return nil, http.ErrSkipAltProtocol + } + return res, err +} diff --git a/vendor/golang.org/x/net/http2/errors.go b/vendor/golang.org/x/net/http2/errors.go index c885328a827..f320b2c09ef 100644 --- a/vendor/golang.org/x/net/http2/errors.go +++ b/vendor/golang.org/x/net/http2/errors.go @@ -1,7 +1,6 @@ -// Copyright 2014 The Go Authors. -// See https://code.google.com/p/go/source/browse/CONTRIBUTORS -// Licensed under the same terms as Go itself: -// https://code.google.com/p/go/source/browse/LICENSE +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. package http2 @@ -76,3 +75,16 @@ func (e StreamError) Error() string { type goAwayFlowError struct{} func (goAwayFlowError) Error() string { return "connection exceeded flow control window size" } + +// connErrorReason wraps a ConnectionError with an informative error about why it occurs. + +// Errors of this type are only returned by the frame parser functions +// and converted into ConnectionError(ErrCodeProtocol). +type connError struct { + Code ErrCode + Reason string +} + +func (e connError) Error() string { + return fmt.Sprintf("http2: connection error: %v: %v", e.Code, e.Reason) +} diff --git a/vendor/golang.org/x/net/http2/fixed_buffer.go b/vendor/golang.org/x/net/http2/fixed_buffer.go new file mode 100644 index 00000000000..47da0f0bf71 --- /dev/null +++ b/vendor/golang.org/x/net/http2/fixed_buffer.go @@ -0,0 +1,60 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package http2 + +import ( + "errors" +) + +// fixedBuffer is an io.ReadWriter backed by a fixed size buffer. +// It never allocates, but moves old data as new data is written. +type fixedBuffer struct { + buf []byte + r, w int +} + +var ( + errReadEmpty = errors.New("read from empty fixedBuffer") + errWriteFull = errors.New("write on full fixedBuffer") +) + +// Read copies bytes from the buffer into p. +// It is an error to read when no data is available. +func (b *fixedBuffer) Read(p []byte) (n int, err error) { + if b.r == b.w { + return 0, errReadEmpty + } + n = copy(p, b.buf[b.r:b.w]) + b.r += n + if b.r == b.w { + b.r = 0 + b.w = 0 + } + return n, nil +} + +// Len returns the number of bytes of the unread portion of the buffer. +func (b *fixedBuffer) Len() int { + return b.w - b.r +} + +// Write copies bytes from p into the buffer. +// It is an error to write more data than the buffer can hold. +func (b *fixedBuffer) Write(p []byte) (n int, err error) { + // Slide existing data to beginning. + if b.r > 0 && len(p) > len(b.buf)-b.w { + copy(b.buf, b.buf[b.r:b.w]) + b.w -= b.r + b.r = 0 + } + + // Write new data. + n = copy(b.buf[b.w:], p) + b.w += n + if n < len(p) { + err = errWriteFull + } + return n, err +} diff --git a/vendor/golang.org/x/net/http2/flow.go b/vendor/golang.org/x/net/http2/flow.go index 540fc4283e5..957de25420d 100644 --- a/vendor/golang.org/x/net/http2/flow.go +++ b/vendor/golang.org/x/net/http2/flow.go @@ -1,7 +1,6 @@ -// Copyright 2014 The Go Authors. -// See https://code.google.com/p/go/source/browse/CONTRIBUTORS -// Licensed under the same terms as Go itself: -// https://code.google.com/p/go/source/browse/LICENSE +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. // Flow control diff --git a/vendor/golang.org/x/net/http2/frame.go b/vendor/golang.org/x/net/http2/frame.go index e8b872a19ba..e1e837cc8a2 100644 --- a/vendor/golang.org/x/net/http2/frame.go +++ b/vendor/golang.org/x/net/http2/frame.go @@ -1,7 +1,6 @@ -// Copyright 2014 The Go Authors. -// See https://code.google.com/p/go/source/browse/CONTRIBUTORS -// Licensed under the same terms as Go itself: -// https://code.google.com/p/go/source/browse/LICENSE +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. package http2 @@ -11,6 +10,7 @@ import ( "errors" "fmt" "io" + "log" "sync" ) @@ -172,6 +172,12 @@ func (h FrameHeader) Header() FrameHeader { return h } func (h FrameHeader) String() string { var buf bytes.Buffer buf.WriteString("[FrameHeader ") + h.writeDebug(&buf) + buf.WriteByte(']') + return buf.String() +} + +func (h FrameHeader) writeDebug(buf *bytes.Buffer) { buf.WriteString(h.Type.String()) if h.Flags != 0 { buf.WriteString(" flags=") @@ -188,15 +194,14 @@ func (h FrameHeader) String() string { if name != "" { buf.WriteString(name) } else { - fmt.Fprintf(&buf, "0x%x", 1<>16), byte(length>>8), byte(length)) + if logFrameWrites { + f.logWrite() + } + n, err := f.w.Write(f.wbuf) if err == nil && n != len(f.wbuf) { err = io.ErrShortWrite @@ -318,6 +343,24 @@ func (f *Framer) endWrite() error { return err } +func (f *Framer) logWrite() { + if f.debugFramer == nil { + f.debugFramerBuf = new(bytes.Buffer) + f.debugFramer = NewFramer(nil, f.debugFramerBuf) + f.debugFramer.logReads = false // we log it ourselves, saying "wrote" below + // Let us read anything, even if we accidentally wrote it + // in the wrong order: + f.debugFramer.AllowIllegalReads = true + } + f.debugFramerBuf.Write(f.wbuf) + fr, err := f.debugFramer.ReadFrame() + if err != nil { + log.Printf("http2: Framer %p: failed to decode just-written frame", f) + return + } + log.Printf("http2: Framer %p: wrote %v", f, summarizeFrame(fr)) +} + func (f *Framer) writeByte(v byte) { f.wbuf = append(f.wbuf, v) } func (f *Framer) writeBytes(v []byte) { f.wbuf = append(f.wbuf, v...) } func (f *Framer) writeUint16(v uint16) { f.wbuf = append(f.wbuf, byte(v>>8), byte(v)) } @@ -333,8 +376,9 @@ const ( // NewFramer returns a Framer that writes frames to w and reads them from r. func NewFramer(w io.Writer, r io.Reader) *Framer { fr := &Framer{ - w: w, - r: r, + w: w, + r: r, + logReads: logFrameReads, } fr.getReadBuf = func(size uint32) []byte { if cap(fr.readBuf) >= int(size) { @@ -362,10 +406,22 @@ func (fr *Framer) SetMaxReadFrameSize(v uint32) { // sends a frame that is larger than declared with SetMaxReadFrameSize. var ErrFrameTooLarge = errors.New("http2: frame too large") +// terminalReadFrameError reports whether err is an unrecoverable +// error from ReadFrame and no other frames should be read. +func terminalReadFrameError(err error) bool { + if _, ok := err.(StreamError); ok { + return false + } + return err != nil +} + // ReadFrame reads a single frame. The returned Frame is only valid // until the next call to ReadFrame. -// If the frame is larger than previously set with SetMaxReadFrameSize, -// the returned error is ErrFrameTooLarge. +// +// If the frame is larger than previously set with SetMaxReadFrameSize, the +// returned error is ErrFrameTooLarge. Other errors may be of type +// ConnectionError, StreamError, or anything else from from the underlying +// reader. func (fr *Framer) ReadFrame() (Frame, error) { if fr.lastFrame != nil { fr.lastFrame.invalidate() @@ -383,12 +439,68 @@ func (fr *Framer) ReadFrame() (Frame, error) { } f, err := typeFrameParser(fh.Type)(fh, payload) if err != nil { + if ce, ok := err.(connError); ok { + return nil, fr.connError(ce.Code, ce.Reason) + } return nil, err } - fr.lastFrame = f + if err := fr.checkFrameOrder(f); err != nil { + return nil, err + } + if fr.logReads { + log.Printf("http2: Framer %p: read %v", fr, summarizeFrame(f)) + } return f, nil } +// connError returns ConnectionError(code) but first +// stashes away a public reason to the caller can optionally relay it +// to the peer before hanging up on them. This might help others debug +// their implementations. +func (fr *Framer) connError(code ErrCode, reason string) error { + fr.errReason = reason + return ConnectionError(code) +} + +// checkFrameOrder reports an error if f is an invalid frame to return +// next from ReadFrame. Mostly it checks whether HEADERS and +// CONTINUATION frames are contiguous. +func (fr *Framer) checkFrameOrder(f Frame) error { + last := fr.lastFrame + fr.lastFrame = f + if fr.AllowIllegalReads { + return nil + } + + fh := f.Header() + if fr.lastHeaderStream != 0 { + if fh.Type != FrameContinuation { + return fr.connError(ErrCodeProtocol, + fmt.Sprintf("got %s for stream %d; expected CONTINUATION following %s for stream %d", + fh.Type, fh.StreamID, + last.Header().Type, fr.lastHeaderStream)) + } + if fh.StreamID != fr.lastHeaderStream { + return fr.connError(ErrCodeProtocol, + fmt.Sprintf("got CONTINUATION for stream %d; expected stream %d", + fh.StreamID, fr.lastHeaderStream)) + } + } else if fh.Type == FrameContinuation { + return fr.connError(ErrCodeProtocol, fmt.Sprintf("unexpected CONTINUATION for stream %d", fh.StreamID)) + } + + switch fh.Type { + case FrameHeaders, FrameContinuation: + if fh.Flags.Has(FlagHeadersEndHeaders) { + fr.lastHeaderStream = 0 + } else { + fr.lastHeaderStream = fh.StreamID + } + } + + return nil +} + // A DataFrame conveys arbitrary, variable-length sequences of octets // associated with a stream. // See http://http2.github.io/http2-spec/#rfc.section.6.1 @@ -417,7 +529,7 @@ func parseDataFrame(fh FrameHeader, payload []byte) (Frame, error) { // field is 0x0, the recipient MUST respond with a // connection error (Section 5.4.1) of type // PROTOCOL_ERROR. - return nil, ConnectionError(ErrCodeProtocol) + return nil, connError{ErrCodeProtocol, "DATA frame with stream ID 0"} } f := &DataFrame{ FrameHeader: fh, @@ -435,7 +547,7 @@ func parseDataFrame(fh FrameHeader, payload []byte) (Frame, error) { // length of the frame payload, the recipient MUST // treat this as a connection error. // Filed: https://github.com/http2/http2-spec/issues/610 - return nil, ConnectionError(ErrCodeProtocol) + return nil, connError{ErrCodeProtocol, "pad size larger than data payload"} } f.data = payload[:len(payload)-int(padSize)] return f, nil @@ -575,6 +687,8 @@ type PingFrame struct { Data [8]byte } +func (f *PingFrame) IsAck() bool { return f.Flags.Has(FlagPingAck) } + func parsePingFrame(fh FrameHeader, payload []byte) (Frame, error) { if len(payload) != 8 { return nil, ConnectionError(ErrCodeFrameSize) @@ -663,7 +777,7 @@ func parseUnknownFrame(fh FrameHeader, p []byte) (Frame, error) { // See http://http2.github.io/http2-spec/#rfc.section.6.9 type WindowUpdateFrame struct { FrameHeader - Increment uint32 + Increment uint32 // never read with high bit set } func parseWindowUpdateFrame(fh FrameHeader, p []byte) (Frame, error) { @@ -740,7 +854,7 @@ func parseHeadersFrame(fh FrameHeader, p []byte) (_ Frame, err error) { // is received whose stream identifier field is 0x0, the recipient MUST // respond with a connection error (Section 5.4.1) of type // PROTOCOL_ERROR. - return nil, ConnectionError(ErrCodeProtocol) + return nil, connError{ErrCodeProtocol, "HEADERS frame with stream ID 0"} } var padLength uint8 if fh.Flags.Has(FlagHeadersPadded) { @@ -870,10 +984,10 @@ func (p PriorityParam) IsZero() bool { func parsePriorityFrame(fh FrameHeader, payload []byte) (Frame, error) { if fh.StreamID == 0 { - return nil, ConnectionError(ErrCodeProtocol) + return nil, connError{ErrCodeProtocol, "PRIORITY frame with stream ID 0"} } if len(payload) != 5 { - return nil, ConnectionError(ErrCodeFrameSize) + return nil, connError{ErrCodeFrameSize, fmt.Sprintf("PRIORITY frame payload size was %d; want 5", len(payload))} } v := binary.BigEndian.Uint32(payload[:4]) streamID := v & 0x7fffffff // mask off high bit @@ -943,13 +1057,12 @@ type ContinuationFrame struct { } func parseContinuationFrame(fh FrameHeader, p []byte) (Frame, error) { + if fh.StreamID == 0 { + return nil, connError{ErrCodeProtocol, "CONTINUATION frame with stream ID 0"} + } return &ContinuationFrame{fh, p}, nil } -func (f *ContinuationFrame) StreamEnded() bool { - return f.FrameHeader.Flags.Has(FlagDataEndStream) -} - func (f *ContinuationFrame) HeaderBlockFragment() []byte { f.checkValid() return f.headerFragBuf @@ -1111,3 +1224,46 @@ type streamEnder interface { type headersEnder interface { HeadersEnded() bool } + +func summarizeFrame(f Frame) string { + var buf bytes.Buffer + f.Header().writeDebug(&buf) + switch f := f.(type) { + case *SettingsFrame: + n := 0 + f.ForeachSetting(func(s Setting) error { + n++ + if n == 1 { + buf.WriteString(", settings:") + } + fmt.Fprintf(&buf, " %v=%v,", s.ID, s.Val) + return nil + }) + if n > 0 { + buf.Truncate(buf.Len() - 1) // remove trailing comma + } + case *DataFrame: + data := f.Data() + const max = 256 + if len(data) > max { + data = data[:max] + } + fmt.Fprintf(&buf, " data=%q", data) + if len(f.Data()) > max { + fmt.Fprintf(&buf, " (%d bytes omitted)", len(f.Data())-max) + } + case *WindowUpdateFrame: + if f.StreamID == 0 { + buf.WriteString(" (conn)") + } + fmt.Fprintf(&buf, " incr=%v", f.Increment) + case *PingFrame: + fmt.Fprintf(&buf, " ping=%q", f.Data[:]) + case *GoAwayFrame: + fmt.Fprintf(&buf, " LastStreamID=%v ErrCode=%v Debug=%q", + f.LastStreamID, f.ErrCode, f.debugData) + case *RSTStreamFrame: + fmt.Fprintf(&buf, " ErrCode=%v", f.ErrCode) + } + return buf.String() +} diff --git a/vendor/golang.org/x/net/http2/go15.go b/vendor/golang.org/x/net/http2/go15.go new file mode 100644 index 00000000000..f0a56241415 --- /dev/null +++ b/vendor/golang.org/x/net/http2/go15.go @@ -0,0 +1,11 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build go1.5 + +package http2 + +import "net/http" + +func requestCancel(req *http.Request) <-chan struct{} { return req.Cancel } diff --git a/vendor/golang.org/x/net/http2/gotrack.go b/vendor/golang.org/x/net/http2/gotrack.go index 7dc2ef90dba..9933c9f8c74 100644 --- a/vendor/golang.org/x/net/http2/gotrack.go +++ b/vendor/golang.org/x/net/http2/gotrack.go @@ -1,9 +1,6 @@ // Copyright 2014 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// See https://code.google.com/p/go/source/browse/CONTRIBUTORS -// Licensed under the same terms as Go itself: -// https://code.google.com/p/go/source/browse/LICENSE // Defensive debug-only utility to track that functions run on the // goroutine that they're supposed to. diff --git a/vendor/golang.org/x/net/http2/headermap.go b/vendor/golang.org/x/net/http2/headermap.go index 67c7c483571..c2805f6ac4d 100644 --- a/vendor/golang.org/x/net/http2/headermap.go +++ b/vendor/golang.org/x/net/http2/headermap.go @@ -1,9 +1,6 @@ // Copyright 2014 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// See https://code.google.com/p/go/source/browse/CONTRIBUTORS -// Licensed under the same terms as Go itself: -// https://code.google.com/p/go/source/browse/LICENSE package http2 @@ -60,6 +57,7 @@ func init() { "server", "set-cookie", "strict-transport-security", + "trailer", "transfer-encoding", "user-agent", "vary", diff --git a/vendor/golang.org/x/net/http2/hpack/encode.go b/vendor/golang.org/x/net/http2/hpack/encode.go index 19bd9f4fcbd..80d621cf355 100644 --- a/vendor/golang.org/x/net/http2/hpack/encode.go +++ b/vendor/golang.org/x/net/http2/hpack/encode.go @@ -1,7 +1,6 @@ -// Copyright 2014 The Go Authors. -// See https://code.google.com/p/go/source/browse/CONTRIBUTORS -// Licensed under the same terms as Go itself: -// https://code.google.com/p/go/source/browse/LICENSE +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. package hpack diff --git a/vendor/golang.org/x/net/http2/hpack/hpack.go b/vendor/golang.org/x/net/http2/hpack/hpack.go index c9e36f74273..2ea4949ab08 100644 --- a/vendor/golang.org/x/net/http2/hpack/hpack.go +++ b/vendor/golang.org/x/net/http2/hpack/hpack.go @@ -1,7 +1,6 @@ -// Copyright 2014 The Go Authors. -// See https://code.google.com/p/go/source/browse/CONTRIBUTORS -// Licensed under the same terms as Go itself: -// https://code.google.com/p/go/source/browse/LICENSE +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. // Package hpack implements HPACK, a compression format for // efficiently representing HTTP header fields in the context of HTTP/2. @@ -42,6 +41,14 @@ type HeaderField struct { Sensitive bool } +func (hf HeaderField) String() string { + var suffix string + if hf.Sensitive { + suffix = " (sensitive)" + } + return fmt.Sprintf("header field %q = %q%s", hf.Name, hf.Value, suffix) +} + func (hf *HeaderField) size() uint32 { // http://http2.github.io/http2-spec/compression.html#rfc.section.4.1 // "The size of the dynamic table is the sum of the size of @@ -64,23 +71,65 @@ type Decoder struct { dynTab dynamicTable emit func(f HeaderField) + emitEnabled bool // whether calls to emit are enabled + maxStrLen int // 0 means unlimited + // buf is the unparsed buffer. It's only written to // saveBuf if it was truncated in the middle of a header // block. Because it's usually not owned, we can only // process it under Write. - buf []byte // usually not owned + buf []byte // not owned; only valid during Write + + // saveBuf is previous data passed to Write which we weren't able + // to fully parse before. Unlike buf, we own this data. saveBuf bytes.Buffer } -func NewDecoder(maxSize uint32, emitFunc func(f HeaderField)) *Decoder { +// NewDecoder returns a new decoder with the provided maximum dynamic +// table size. The emitFunc will be called for each valid field +// parsed, in the same goroutine as calls to Write, before Write returns. +func NewDecoder(maxDynamicTableSize uint32, emitFunc func(f HeaderField)) *Decoder { d := &Decoder{ - emit: emitFunc, + emit: emitFunc, + emitEnabled: true, } - d.dynTab.allowedMaxSize = maxSize - d.dynTab.setMaxSize(maxSize) + d.dynTab.allowedMaxSize = maxDynamicTableSize + d.dynTab.setMaxSize(maxDynamicTableSize) return d } +// ErrStringLength is returned by Decoder.Write when the max string length +// (as configured by Decoder.SetMaxStringLength) would be violated. +var ErrStringLength = errors.New("hpack: string too long") + +// SetMaxStringLength sets the maximum size of a HeaderField name or +// value string. If a string exceeds this length (even after any +// decompression), Write will return ErrStringLength. +// A value of 0 means unlimited and is the default from NewDecoder. +func (d *Decoder) SetMaxStringLength(n int) { + d.maxStrLen = n +} + +// SetEmitFunc changes the callback used when new header fields +// are decoded. +// It must be non-nil. It does not affect EmitEnabled. +func (d *Decoder) SetEmitFunc(emitFunc func(f HeaderField)) { + d.emit = emitFunc +} + +// SetEmitEnabled controls whether the emitFunc provided to NewDecoder +// should be called. The default is true. +// +// This facility exists to let servers enforce MAX_HEADER_LIST_SIZE +// while still decoding and keeping in-sync with decoder state, but +// without doing unnecessary decompression or generating unnecessary +// garbage for header fields past the limit. +func (d *Decoder) SetEmitEnabled(v bool) { d.emitEnabled = v } + +// EmitEnabled reports whether calls to the emitFunc provided to NewDecoder +// are currently enabled. The default is true. +func (d *Decoder) EmitEnabled() bool { return d.emitEnabled } + // TODO: add method *Decoder.Reset(maxSize, emitFunc) to let callers re-use Decoders and their // underlying buffers for garbage reasons. @@ -247,15 +296,23 @@ func (d *Decoder) Write(p []byte) (n int, err error) { for len(d.buf) > 0 { err = d.parseHeaderFieldRepr() - if err != nil { - if err == errNeedMore { - err = nil - d.saveBuf.Write(d.buf) + if err == errNeedMore { + // Extra paranoia, making sure saveBuf won't + // get too large. All the varint and string + // reading code earlier should already catch + // overlong things and return ErrStringLength, + // but keep this as a last resort. + const varIntOverhead = 8 // conservative + if d.maxStrLen != 0 && int64(len(d.buf)) > 2*(int64(d.maxStrLen)+varIntOverhead) { + return 0, ErrStringLength } + d.saveBuf.Write(d.buf) + return len(p), nil + } + if err != nil { break } } - return len(p), err } @@ -323,9 +380,8 @@ func (d *Decoder) parseFieldIndexed() error { if !ok { return DecodingError{InvalidIndexError(idx)} } - d.emit(HeaderField{Name: hf.Name, Value: hf.Value}) d.buf = buf - return nil + return d.callEmit(HeaderField{Name: hf.Name, Value: hf.Value}) } // (same invariants and behavior as parseHeaderFieldRepr) @@ -337,6 +393,7 @@ func (d *Decoder) parseFieldLiteral(n uint8, it indexType) error { } var hf HeaderField + wantStr := d.emitEnabled || it.indexed() if nameIdx > 0 { ihf, ok := d.at(nameIdx) if !ok { @@ -344,12 +401,12 @@ func (d *Decoder) parseFieldLiteral(n uint8, it indexType) error { } hf.Name = ihf.Name } else { - hf.Name, buf, err = readString(buf) + hf.Name, buf, err = d.readString(buf, wantStr) if err != nil { return err } } - hf.Value, buf, err = readString(buf) + hf.Value, buf, err = d.readString(buf, wantStr) if err != nil { return err } @@ -358,7 +415,18 @@ func (d *Decoder) parseFieldLiteral(n uint8, it indexType) error { d.dynTab.add(hf) } hf.Sensitive = it.sensitive() - d.emit(hf) + return d.callEmit(hf) +} + +func (d *Decoder) callEmit(hf HeaderField) error { + if d.maxStrLen != 0 { + if len(hf.Name) > d.maxStrLen || len(hf.Value) > d.maxStrLen { + return ErrStringLength + } + } + if d.emitEnabled { + d.emit(hf) + } return nil } @@ -420,7 +488,15 @@ func readVarInt(n byte, p []byte) (i uint64, remain []byte, err error) { return 0, origP, errNeedMore } -func readString(p []byte) (s string, remain []byte, err error) { +// readString decodes an hpack string from p. +// +// wantStr is whether s will be used. If false, decompression and +// []byte->string garbage are skipped if s will be ignored +// anyway. This does mean that huffman decoding errors for non-indexed +// strings past the MAX_HEADER_LIST_SIZE are ignored, but the server +// is returning an error anyway, and because they're not indexed, the error +// won't affect the decoding state. +func (d *Decoder) readString(p []byte, wantStr bool) (s string, remain []byte, err error) { if len(p) == 0 { return "", p, errNeedMore } @@ -429,17 +505,29 @@ func readString(p []byte) (s string, remain []byte, err error) { if err != nil { return "", p, err } + if d.maxStrLen != 0 && strLen > uint64(d.maxStrLen) { + return "", nil, ErrStringLength + } if uint64(len(p)) < strLen { return "", p, errNeedMore } if !isHuff { - return string(p[:strLen]), p[strLen:], nil + if wantStr { + s = string(p[:strLen]) + } + return s, p[strLen:], nil } - // TODO: optimize this garbage: - var buf bytes.Buffer - if _, err := HuffmanDecode(&buf, p[:strLen]); err != nil { - return "", nil, err + if wantStr { + buf := bufPool.Get().(*bytes.Buffer) + buf.Reset() // don't trust others + defer bufPool.Put(buf) + if err := huffmanDecode(buf, d.maxStrLen, p[:strLen]); err != nil { + buf.Reset() + return "", nil, err + } + s = buf.String() + buf.Reset() // be nice to GC } - return buf.String(), p[strLen:], nil + return s, p[strLen:], nil } diff --git a/vendor/golang.org/x/net/http2/hpack/huffman.go b/vendor/golang.org/x/net/http2/hpack/huffman.go index 9fe76f68ee0..eb4b1f05cd0 100644 --- a/vendor/golang.org/x/net/http2/hpack/huffman.go +++ b/vendor/golang.org/x/net/http2/hpack/huffman.go @@ -1,12 +1,12 @@ -// Copyright 2014 The Go Authors. -// See https://code.google.com/p/go/source/browse/CONTRIBUTORS -// Licensed under the same terms as Go itself: -// https://code.google.com/p/go/source/browse/LICENSE +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. package hpack import ( "bytes" + "errors" "io" "sync" ) @@ -22,15 +22,46 @@ func HuffmanDecode(w io.Writer, v []byte) (int, error) { buf := bufPool.Get().(*bytes.Buffer) buf.Reset() defer bufPool.Put(buf) + if err := huffmanDecode(buf, 0, v); err != nil { + return 0, err + } + return w.Write(buf.Bytes()) +} +// HuffmanDecodeToString decodes the string in v. +func HuffmanDecodeToString(v []byte) (string, error) { + buf := bufPool.Get().(*bytes.Buffer) + buf.Reset() + defer bufPool.Put(buf) + if err := huffmanDecode(buf, 0, v); err != nil { + return "", err + } + return buf.String(), nil +} + +// ErrInvalidHuffman is returned for errors found decoding +// Huffman-encoded strings. +var ErrInvalidHuffman = errors.New("hpack: invalid Huffman-encoded data") + +// huffmanDecode decodes v to buf. +// If maxLen is greater than 0, attempts to write more to buf than +// maxLen bytes will return ErrStringLength. +func huffmanDecode(buf *bytes.Buffer, maxLen int, v []byte) error { n := rootHuffmanNode cur, nbits := uint(0), uint8(0) for _, b := range v { cur = cur<<8 | uint(b) nbits += 8 for nbits >= 8 { - n = n.children[byte(cur>>(nbits-8))] + idx := byte(cur >> (nbits - 8)) + n = n.children[idx] + if n == nil { + return ErrInvalidHuffman + } if n.children == nil { + if maxLen != 0 && buf.Len() == maxLen { + return ErrStringLength + } buf.WriteByte(n.sym) nbits -= n.codeLen n = rootHuffmanNode @@ -48,7 +79,7 @@ func HuffmanDecode(w io.Writer, v []byte) (int, error) { nbits -= n.codeLen n = rootHuffmanNode } - return w.Write(buf.Bytes()) + return nil } type node struct { @@ -67,10 +98,10 @@ func newInternalNode() *node { var rootHuffmanNode = newInternalNode() func init() { + if len(huffmanCodes) != 256 { + panic("unexpected size") + } for i, code := range huffmanCodes { - if i > 255 { - panic("too many huffman codes") - } addDecoderNode(byte(i), code, huffmanCodeLen[i]) } } diff --git a/vendor/golang.org/x/net/http2/hpack/tables.go b/vendor/golang.org/x/net/http2/hpack/tables.go index f898e251267..b9283a02330 100644 --- a/vendor/golang.org/x/net/http2/hpack/tables.go +++ b/vendor/golang.org/x/net/http2/hpack/tables.go @@ -1,7 +1,6 @@ -// Copyright 2014 The Go Authors. -// See https://code.google.com/p/go/source/browse/CONTRIBUTORS -// Licensed under the same terms as Go itself: -// https://code.google.com/p/go/source/browse/LICENSE +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. package hpack @@ -10,7 +9,7 @@ func pair(name, value string) HeaderField { } // http://tools.ietf.org/html/draft-ietf-httpbis-header-compression-07#appendix-B -var staticTable = []HeaderField{ +var staticTable = [...]HeaderField{ pair(":authority", ""), // index 1 (1-based) pair(":method", "GET"), pair(":method", "POST"), @@ -74,7 +73,7 @@ var staticTable = []HeaderField{ pair("www-authenticate", ""), } -var huffmanCodes = []uint32{ +var huffmanCodes = [256]uint32{ 0x1ff8, 0x7fffd8, 0xfffffe2, @@ -333,7 +332,7 @@ var huffmanCodes = []uint32{ 0x3ffffee, } -var huffmanCodeLen = []uint8{ +var huffmanCodeLen = [256]uint8{ 13, 23, 28, 28, 28, 28, 28, 28, 28, 24, 30, 28, 28, 30, 28, 28, 28, 28, 28, 28, 28, 28, 30, 28, 28, 28, 28, 28, 28, 28, 28, 28, 6, 10, 10, 12, 13, 6, 8, 11, 10, 10, 8, 11, 8, 6, 6, 6, diff --git a/vendor/golang.org/x/net/http2/http2.go b/vendor/golang.org/x/net/http2/http2.go index 35f9b26e282..4c5e11ac2cd 100644 --- a/vendor/golang.org/x/net/http2/http2.go +++ b/vendor/golang.org/x/net/http2/http2.go @@ -1,31 +1,50 @@ // Copyright 2014 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// See https://code.google.com/p/go/source/browse/CONTRIBUTORS -// Licensed under the same terms as Go itself: -// https://code.google.com/p/go/source/browse/LICENSE // Package http2 implements the HTTP/2 protocol. // -// This is a work in progress. This package is low-level and intended -// to be used directly by very few people. Most users will use it -// indirectly through integration with the net/http package. See -// ConfigureServer. That ConfigureServer call will likely be automatic -// or available via an empty import in the future. +// This package is low-level and intended to be used directly by very +// few people. Most users will use it indirectly through the automatic +// use by the net/http package (from Go 1.6 and later). +// For use in earlier Go versions see ConfigureServer. (Transport support +// requires Go 1.6 or later) // -// See http://http2.github.io/ +// See https://http2.github.io/ for more information on HTTP/2. +// +// See https://http2.golang.org/ for a test server running this code. package http2 import ( "bufio" + "crypto/tls" + "errors" "fmt" "io" "net/http" + "os" "strconv" + "strings" "sync" ) -var VerboseLogs = false +var ( + VerboseLogs bool + logFrameWrites bool + logFrameReads bool +) + +func init() { + e := os.Getenv("GODEBUG") + if strings.Contains(e, "http2debug=1") { + VerboseLogs = true + } + if strings.Contains(e, "http2debug=2") { + VerboseLogs = true + logFrameWrites = true + logFrameReads = true + } +} const ( // ClientPreface is the string that must be sent by new @@ -141,17 +160,62 @@ func (s SettingID) String() string { return fmt.Sprintf("UNKNOWN_SETTING_%d", uint16(s)) } -func validHeader(v string) bool { +var ( + errInvalidHeaderFieldName = errors.New("http2: invalid header field name") + errInvalidHeaderFieldValue = errors.New("http2: invalid header field value") +) + +// validHeaderFieldName reports whether v is a valid header field name (key). +// RFC 7230 says: +// header-field = field-name ":" OWS field-value OWS +// field-name = token +// tchar = "!" / "#" / "$" / "%" / "&" / "'" / "*" / "+" / "-" / "." / +// "^" / "_" / " +// Further, http2 says: +// "Just as in HTTP/1.x, header field names are strings of ASCII +// characters that are compared in a case-insensitive +// fashion. However, header field names MUST be converted to +// lowercase prior to their encoding in HTTP/2. " +func validHeaderFieldName(v string) bool { if len(v) == 0 { return false } for _, r := range v { - // "Just as in HTTP/1.x, header field names are - // strings of ASCII characters that are compared in a - // case-insensitive fashion. However, header field - // names MUST be converted to lowercase prior to their - // encoding in HTTP/2. " - if r >= 127 || ('A' <= r && r <= 'Z') { + if int(r) >= len(isTokenTable) || ('A' <= r && r <= 'Z') { + return false + } + if !isTokenTable[byte(r)] { + return false + } + } + return true +} + +// validHeaderFieldValue reports whether v is a valid header field value. +// +// RFC 7230 says: +// field-value = *( field-content / obs-fold ) +// obj-fold = N/A to http2, and deprecated +// field-content = field-vchar [ 1*( SP / HTAB ) field-vchar ] +// field-vchar = VCHAR / obs-text +// obs-text = %x80-FF +// VCHAR = "any visible [USASCII] character" +// +// http2 further says: "Similarly, HTTP/2 allows header field values +// that are not valid. While most of the values that can be encoded +// will not alter header field parsing, carriage return (CR, ASCII +// 0xd), line feed (LF, ASCII 0xa), and the zero character (NUL, ASCII +// 0x0) might be exploited by an attacker if they are translated +// verbatim. Any request or response that contains a character not +// permitted in a header field value MUST be treated as malformed +// (Section 8.1.2.6). Valid characters are defined by the +// field-content ABNF rule in Section 3.2 of [RFC7230]." +// +// This function does not (yet?) properly handle the rejection of +// strings that begin or end with SP or HTAB. +func validHeaderFieldValue(v string) bool { + for i := 0; i < len(v); i++ { + if b := v[i]; b < ' ' && b != '\t' || b == 0x7f { return false } } @@ -247,3 +311,119 @@ func (w *bufferedWriter) Flush() error { w.bw = nil return err } + +func mustUint31(v int32) uint32 { + if v < 0 || v > 2147483647 { + panic("out of range") + } + return uint32(v) +} + +// bodyAllowedForStatus reports whether a given response status code +// permits a body. See RFC2616, section 4.4. +func bodyAllowedForStatus(status int) bool { + switch { + case status >= 100 && status <= 199: + return false + case status == 204: + return false + case status == 304: + return false + } + return true +} + +type httpError struct { + msg string + timeout bool +} + +func (e *httpError) Error() string { return e.msg } +func (e *httpError) Timeout() bool { return e.timeout } +func (e *httpError) Temporary() bool { return true } + +var errTimeout error = &httpError{msg: "http2: timeout awaiting response headers", timeout: true} + +var isTokenTable = [127]bool{ + '!': true, + '#': true, + '$': true, + '%': true, + '&': true, + '\'': true, + '*': true, + '+': true, + '-': true, + '.': true, + '0': true, + '1': true, + '2': true, + '3': true, + '4': true, + '5': true, + '6': true, + '7': true, + '8': true, + '9': true, + 'A': true, + 'B': true, + 'C': true, + 'D': true, + 'E': true, + 'F': true, + 'G': true, + 'H': true, + 'I': true, + 'J': true, + 'K': true, + 'L': true, + 'M': true, + 'N': true, + 'O': true, + 'P': true, + 'Q': true, + 'R': true, + 'S': true, + 'T': true, + 'U': true, + 'W': true, + 'V': true, + 'X': true, + 'Y': true, + 'Z': true, + '^': true, + '_': true, + '`': true, + 'a': true, + 'b': true, + 'c': true, + 'd': true, + 'e': true, + 'f': true, + 'g': true, + 'h': true, + 'i': true, + 'j': true, + 'k': true, + 'l': true, + 'm': true, + 'n': true, + 'o': true, + 'p': true, + 'q': true, + 'r': true, + 's': true, + 't': true, + 'u': true, + 'v': true, + 'w': true, + 'x': true, + 'y': true, + 'z': true, + '|': true, + '~': true, +} + +type connectionStater interface { + ConnectionState() tls.ConnectionState +} diff --git a/vendor/golang.org/x/net/http2/not_go15.go b/vendor/golang.org/x/net/http2/not_go15.go new file mode 100644 index 00000000000..d0fa5c89069 --- /dev/null +++ b/vendor/golang.org/x/net/http2/not_go15.go @@ -0,0 +1,11 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !go1.5 + +package http2 + +import "net/http" + +func requestCancel(req *http.Request) <-chan struct{} { return nil } diff --git a/vendor/golang.org/x/net/http2/not_go16.go b/vendor/golang.org/x/net/http2/not_go16.go new file mode 100644 index 00000000000..db53c5b8cbf --- /dev/null +++ b/vendor/golang.org/x/net/http2/not_go16.go @@ -0,0 +1,13 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !go1.6 + +package http2 + +import "net/http" + +func configureTransport(t1 *http.Transport) (*Transport, error) { + return nil, errTransportVersion +} diff --git a/vendor/golang.org/x/net/http2/pipe.go b/vendor/golang.org/x/net/http2/pipe.go index ce9aad53364..69446e7a370 100644 --- a/vendor/golang.org/x/net/http2/pipe.go +++ b/vendor/golang.org/x/net/http2/pipe.go @@ -1,43 +1,147 @@ -// Copyright 2014 The Go Authors. -// See https://code.google.com/p/go/source/browse/CONTRIBUTORS -// Licensed under the same terms as Go itself: -// https://code.google.com/p/go/source/browse/LICENSE +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. package http2 import ( + "errors" + "io" "sync" ) +// pipe is a goroutine-safe io.Reader/io.Writer pair. It's like +// io.Pipe except there are no PipeReader/PipeWriter halves, and the +// underlying buffer is an interface. (io.Pipe is always unbuffered) type pipe struct { - b buffer - c sync.Cond - m sync.Mutex + mu sync.Mutex + c sync.Cond // c.L lazily initialized to &p.mu + b pipeBuffer + err error // read error once empty. non-nil means closed. + breakErr error // immediate read error (caller doesn't see rest of b) + donec chan struct{} // closed on error + readFn func() // optional code to run in Read before error +} + +type pipeBuffer interface { + Len() int + io.Writer + io.Reader } // Read waits until data is available and copies bytes // from the buffer into p. -func (r *pipe) Read(p []byte) (n int, err error) { - r.c.L.Lock() - defer r.c.L.Unlock() - for r.b.Len() == 0 && !r.b.closed { - r.c.Wait() +func (p *pipe) Read(d []byte) (n int, err error) { + p.mu.Lock() + defer p.mu.Unlock() + if p.c.L == nil { + p.c.L = &p.mu + } + for { + if p.breakErr != nil { + return 0, p.breakErr + } + if p.b.Len() > 0 { + return p.b.Read(d) + } + if p.err != nil { + if p.readFn != nil { + p.readFn() // e.g. copy trailers + p.readFn = nil // not sticky like p.err + } + return 0, p.err + } + p.c.Wait() } - return r.b.Read(p) } +var errClosedPipeWrite = errors.New("write on closed buffer") + // Write copies bytes from p into the buffer and wakes a reader. // It is an error to write more data than the buffer can hold. -func (w *pipe) Write(p []byte) (n int, err error) { - w.c.L.Lock() - defer w.c.L.Unlock() - defer w.c.Signal() - return w.b.Write(p) +func (p *pipe) Write(d []byte) (n int, err error) { + p.mu.Lock() + defer p.mu.Unlock() + if p.c.L == nil { + p.c.L = &p.mu + } + defer p.c.Signal() + if p.err != nil { + return 0, errClosedPipeWrite + } + return p.b.Write(d) } -func (c *pipe) Close(err error) { - c.c.L.Lock() - defer c.c.L.Unlock() - defer c.c.Signal() - c.b.Close(err) +// CloseWithError causes the next Read (waking up a current blocked +// Read if needed) to return the provided err after all data has been +// read. +// +// The error must be non-nil. +func (p *pipe) CloseWithError(err error) { p.closeWithError(&p.err, err, nil) } + +// BreakWithError causes the next Read (waking up a current blocked +// Read if needed) to return the provided err immediately, without +// waiting for unread data. +func (p *pipe) BreakWithError(err error) { p.closeWithError(&p.breakErr, err, nil) } + +// closeWithErrorAndCode is like CloseWithError but also sets some code to run +// in the caller's goroutine before returning the error. +func (p *pipe) closeWithErrorAndCode(err error, fn func()) { p.closeWithError(&p.err, err, fn) } + +func (p *pipe) closeWithError(dst *error, err error, fn func()) { + if err == nil { + panic("err must be non-nil") + } + p.mu.Lock() + defer p.mu.Unlock() + if p.c.L == nil { + p.c.L = &p.mu + } + defer p.c.Signal() + if *dst != nil { + // Already been done. + return + } + p.readFn = fn + *dst = err + p.closeDoneLocked() +} + +// requires p.mu be held. +func (p *pipe) closeDoneLocked() { + if p.donec == nil { + return + } + // Close if unclosed. This isn't racy since we always + // hold p.mu while closing. + select { + case <-p.donec: + default: + close(p.donec) + } +} + +// Err returns the error (if any) first set by BreakWithError or CloseWithError. +func (p *pipe) Err() error { + p.mu.Lock() + defer p.mu.Unlock() + if p.breakErr != nil { + return p.breakErr + } + return p.err +} + +// Done returns a channel which is closed if and when this pipe is closed +// with CloseWithError. +func (p *pipe) Done() <-chan struct{} { + p.mu.Lock() + defer p.mu.Unlock() + if p.donec == nil { + p.donec = make(chan struct{}) + if p.err != nil || p.breakErr != nil { + // Already hit an error. + p.closeDoneLocked() + } + } + return p.donec } diff --git a/vendor/golang.org/x/net/http2/server.go b/vendor/golang.org/x/net/http2/server.go index 99cc673cc2e..6f4c2bb7cd9 100644 --- a/vendor/golang.org/x/net/http2/server.go +++ b/vendor/golang.org/x/net/http2/server.go @@ -1,16 +1,13 @@ // Copyright 2014 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// See https://code.google.com/p/go/source/browse/CONTRIBUTORS -// Licensed under the same terms as Go itself: -// https://code.google.com/p/go/source/browse/LICENSE // TODO: replace all <-sc.doneServing with reads from the stream's cw // instead, and make sure that on close we close all open // streams. then remove doneServing? -// TODO: finish GOAWAY support. Consider each incoming frame type and -// whether it should be ignored during a shutdown race. +// TODO: re-audit GOAWAY support. Consider each incoming frame type and +// whether it should be ignored during graceful shutdown. // TODO: disconnect idle clients. GFE seems to do 4 minutes. make // configurable? or maximum number of idle clients and remove the @@ -49,7 +46,12 @@ import ( "log" "net" "net/http" + "net/textproto" "net/url" + "os" + "reflect" + "runtime" + "sort" "strconv" "strings" "sync" @@ -68,7 +70,8 @@ const ( var ( errClientDisconnected = errors.New("client disconnected") errClosedBody = errors.New("body closed by handler") - errStreamBroken = errors.New("http2: stream broken") + errHandlerComplete = errors.New("http2: request body closed due to handler exiting") + errStreamClosed = errors.New("http2: stream closed") ) var responseWriterStatePool = sync.Pool{ @@ -133,12 +136,33 @@ func (s *Server) maxConcurrentStreams() uint32 { // The configuration conf may be nil. // // ConfigureServer must be called before s begins serving. -func ConfigureServer(s *http.Server, conf *Server) { +func ConfigureServer(s *http.Server, conf *Server) error { if conf == nil { conf = new(Server) } + if s.TLSConfig == nil { s.TLSConfig = new(tls.Config) + } else if s.TLSConfig.CipherSuites != nil { + // If they already provided a CipherSuite list, return + // an error if it has a bad order or is missing + // ECDHE_RSA_WITH_AES_128_GCM_SHA256. + const requiredCipher = tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + haveRequired := false + sawBad := false + for i, cs := range s.TLSConfig.CipherSuites { + if cs == requiredCipher { + haveRequired = true + } + if isBadCipher(cs) { + sawBad = true + } else if sawBad { + return fmt.Errorf("http2: TLSConfig.CipherSuites index %d contains an HTTP/2-approved cipher suite (%#04x), but it comes after unapproved cipher suites. With this configuration, clients that don't support previous, approved cipher suites may be given an unapproved one and reject the connection.", i, cs) + } + } + if !haveRequired { + return fmt.Errorf("http2: TLSConfig.CipherSuites is missing HTTP/2-required TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256") + } } // Note: not setting MinVersion to tls.VersionTLS12, @@ -148,22 +172,7 @@ func ConfigureServer(s *http.Server, conf *Server) { // during next-proto selection, but using TLS <1.2 with // HTTP/2 is still the client's bug. - // Be sure we advertise tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - // at least. - // TODO: enable PreferServerCipherSuites? - if s.TLSConfig.CipherSuites != nil { - const requiredCipher = tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - haveRequired := false - for _, v := range s.TLSConfig.CipherSuites { - if v == requiredCipher { - haveRequired = true - break - } - } - if !haveRequired { - s.TLSConfig.CipherSuites = append(s.TLSConfig.CipherSuites, requiredCipher) - } - } + s.TLSConfig.PreferServerCipherSuites = true haveNPN := false for _, p := range s.TLSConfig.NextProtos { @@ -186,28 +195,76 @@ func ConfigureServer(s *http.Server, conf *Server) { if testHookOnConn != nil { testHookOnConn() } - conf.handleConn(hs, c, h) + conf.ServeConn(c, &ServeConnOpts{ + Handler: h, + BaseConfig: hs, + }) } s.TLSNextProto[NextProtoTLS] = protoHandler s.TLSNextProto["h2-14"] = protoHandler // temporary; see above. + return nil } -func (srv *Server) handleConn(hs *http.Server, c net.Conn, h http.Handler) { +// ServeConnOpts are options for the Server.ServeConn method. +type ServeConnOpts struct { + // BaseConfig optionally sets the base configuration + // for values. If nil, defaults are used. + BaseConfig *http.Server + + // Handler specifies which handler to use for processing + // requests. If nil, BaseConfig.Handler is used. If BaseConfig + // or BaseConfig.Handler is nil, http.DefaultServeMux is used. + Handler http.Handler +} + +func (o *ServeConnOpts) baseConfig() *http.Server { + if o != nil && o.BaseConfig != nil { + return o.BaseConfig + } + return new(http.Server) +} + +func (o *ServeConnOpts) handler() http.Handler { + if o != nil { + if o.Handler != nil { + return o.Handler + } + if o.BaseConfig != nil && o.BaseConfig.Handler != nil { + return o.BaseConfig.Handler + } + } + return http.DefaultServeMux +} + +// ServeConn serves HTTP/2 requests on the provided connection and +// blocks until the connection is no longer readable. +// +// ServeConn starts speaking HTTP/2 assuming that c has not had any +// reads or writes. It writes its initial settings frame and expects +// to be able to read the preface and settings frame from the +// client. If c has a ConnectionState method like a *tls.Conn, the +// ConnectionState is used to verify the TLS ciphersuite and to set +// the Request.TLS field in Handlers. +// +// ServeConn does not support h2c by itself. Any h2c support must be +// implemented in terms of providing a suitably-behaving net.Conn. +// +// The opts parameter is optional. If nil, default values are used. +func (s *Server) ServeConn(c net.Conn, opts *ServeConnOpts) { sc := &serverConn{ - srv: srv, - hs: hs, + srv: s, + hs: opts.baseConfig(), conn: c, remoteAddrStr: c.RemoteAddr().String(), bw: newBufferedWriter(c), - handler: h, + handler: opts.handler(), streams: make(map[uint32]*stream), - readFrameCh: make(chan frameAndGate), - readFrameErrCh: make(chan error, 1), // must be buffered for 1 + readFrameCh: make(chan readFrameResult), wantWriteFrameCh: make(chan frameWriteMsg, 8), - wroteFrameCh: make(chan struct{}, 1), // buffered; one send in reading goroutine - bodyReadCh: make(chan bodyReadMsg), // buffering doesn't matter either way + wroteFrameCh: make(chan frameWriteResult, 1), // buffered; one send in writeFrameAsync + bodyReadCh: make(chan bodyReadMsg), // buffering doesn't matter either way doneServing: make(chan struct{}), - advMaxStreams: srv.maxConcurrentStreams(), + advMaxStreams: s.maxConcurrentStreams(), writeSched: writeScheduler{ maxFrameSize: initialMaxFrameSize, }, @@ -219,13 +276,14 @@ func (srv *Server) handleConn(hs *http.Server, c net.Conn, h http.Handler) { sc.flow.add(initialWindowSize) sc.inflow.add(initialWindowSize) sc.hpackEncoder = hpack.NewEncoder(&sc.headerWriteBuf) - sc.hpackDecoder = hpack.NewDecoder(initialHeaderTableSize, sc.onNewHeaderField) + sc.hpackDecoder = hpack.NewDecoder(initialHeaderTableSize, nil) + sc.hpackDecoder.SetMaxStringLength(sc.maxHeaderStringLen()) fr := NewFramer(sc.bw, c) - fr.SetMaxReadFrameSize(srv.maxReadFrameSize()) + fr.SetMaxReadFrameSize(s.maxReadFrameSize()) sc.framer = fr - if tc, ok := c.(*tls.Conn); ok { + if tc, ok := c.(connectionStater); ok { sc.tlsState = new(tls.ConnectionState) *sc.tlsState = tc.ConnectionState() // 9.2 Use of TLS Features @@ -255,7 +313,7 @@ func (srv *Server) handleConn(hs *http.Server, c net.Conn, h http.Handler) { // So for now, do nothing here again. } - if !srv.PermitProhibitedCipherSuites && isBadCipher(sc.tlsState.CipherSuite) { + if !s.PermitProhibitedCipherSuites && isBadCipher(sc.tlsState.CipherSuite) { // "Endpoints MAY choose to generate a connection error // (Section 5.4.1) of type INADEQUATE_SECURITY if one of // the prohibited cipher suites are negotiated." @@ -302,23 +360,13 @@ func isBadCipher(cipher uint16) bool { } func (sc *serverConn) rejectConn(err ErrCode, debug string) { - log.Printf("REJECTING conn: %v, %s", err, debug) + sc.vlogf("http2: server rejecting conn: %v, %s", err, debug) // ignoring errors. hanging up anyway. sc.framer.WriteGoAway(0, err, []byte(debug)) sc.bw.Flush() sc.conn.Close() } -// frameAndGates coordinates the readFrames and serve -// goroutines. Because the Framer interface only permits the most -// recently-read Frame from being accessed, the readFrames goroutine -// blocks until it has a frame, passes it to serve, and then waits for -// serve to be done with it before reading the next one. -type frameAndGate struct { - f Frame - g gate -} - type serverConn struct { // Immutable: srv *Server @@ -328,16 +376,15 @@ type serverConn struct { handler http.Handler framer *Framer hpackDecoder *hpack.Decoder - doneServing chan struct{} // closed when serverConn.serve ends - readFrameCh chan frameAndGate // written by serverConn.readFrames - readFrameErrCh chan error - wantWriteFrameCh chan frameWriteMsg // from handlers -> serve - wroteFrameCh chan struct{} // from writeFrameAsync -> serve, tickles more frame writes - bodyReadCh chan bodyReadMsg // from handlers -> serve - testHookCh chan func() // code to run on the serve loop - flow flow // conn-wide (not stream-specific) outbound flow control - inflow flow // conn-wide inbound flow control - tlsState *tls.ConnectionState // shared by all handlers, like net/http + doneServing chan struct{} // closed when serverConn.serve ends + readFrameCh chan readFrameResult // written by serverConn.readFrames + wantWriteFrameCh chan frameWriteMsg // from handlers -> serve + wroteFrameCh chan frameWriteResult // from writeFrameAsync -> serve, tickles more frame writes + bodyReadCh chan bodyReadMsg // from handlers -> serve + testHookCh chan func(int) // code to run on the serve loop + flow flow // conn-wide (not stream-specific) outbound flow control + inflow flow // conn-wide inbound flow control + tlsState *tls.ConnectionState // shared by all handlers, like net/http remoteAddrStr string // Everything following is owned by the serve loop; use serveG.check(): @@ -353,7 +400,7 @@ type serverConn struct { streams map[uint32]*stream initialWindowSize int32 headerTableSize uint32 - maxHeaderListSize uint32 // zero means unknown (default) + peerMaxHeaderListSize uint32 // zero means unknown (default) canonHeader map[string]string // http2-lower-case -> Go-Canonical-Case req requestParam // non-zero while reading request headers writingFrame bool // started write goroutine but haven't heard back on wroteFrameCh @@ -370,6 +417,28 @@ type serverConn struct { hpackEncoder *hpack.Encoder } +func (sc *serverConn) maxHeaderStringLen() int { + v := sc.maxHeaderListSize() + if uint32(int(v)) == v { + return int(v) + } + // They had a crazy big number for MaxHeaderBytes anyway, + // so give them unlimited header lengths: + return 0 +} + +func (sc *serverConn) maxHeaderListSize() uint32 { + n := sc.hs.MaxHeaderBytes + if n <= 0 { + n = http.DefaultMaxHeaderBytes + } + // http2's count is in a slightly different unit and includes 32 bytes per pair. + // So, take the net/http.Server value and pad it up a bit, assuming 10 headers. + const perFieldOverhead = 32 // per http2 spec + const typicalHeaders = 10 // conservative + return uint32(n + typicalHeaders*perFieldOverhead) +} + // requestParam is the state of the next request, initialized over // potentially several frames HEADERS + zero or more CONTINUATION // frames. @@ -380,8 +449,9 @@ type requestParam struct { header http.Header method, path string scheme, authority string - sawRegularHeader bool // saw a non-pseudo header already - invalidHeader bool // an invalid header was seen + sawRegularHeader bool // saw a non-pseudo header already + invalidHeader bool // an invalid header was seen + headerListSize int64 // actually uint32, but easier math this way } // stream represents a stream. This is the minimal metadata needed by @@ -393,20 +463,26 @@ type requestParam struct { // responseWriter's state field. type stream struct { // immutable: + sc *serverConn id uint32 body *pipe // non-nil if expecting DATA frames cw closeWaiter // closed wait stream transitions to closed state // owned by serverConn's serve loop: - bodyBytes int64 // body bytes seen so far - declBodyBytes int64 // or -1 if undeclared - flow flow // limits writing from Handler to client - inflow flow // what the client is allowed to POST/etc to us - parent *stream // or nil - weight uint8 - state streamState - sentReset bool // only true once detached from streams map - gotReset bool // only true once detacted from streams map + bodyBytes int64 // body bytes seen so far + declBodyBytes int64 // or -1 if undeclared + flow flow // limits writing from Handler to client + inflow flow // what the client is allowed to POST/etc to us + parent *stream // or nil + numTrailerValues int64 + weight uint8 + state streamState + sentReset bool // only true once detached from streams map + gotReset bool // only true once detacted from streams map + gotTrailerHeader bool // HEADER frame for trailers was seen + + trailer http.Header // accumulated trailers + reqTrailer http.Header // handler's Request.Trailer } func (sc *serverConn) Framer() *Framer { return sc.framer } @@ -434,6 +510,15 @@ func (sc *serverConn) state(streamID uint32) (streamState, *stream) { return stateIdle, nil } +// setConnState calls the net/http ConnState hook for this connection, if configured. +// Note that the net/http package does StateNew and StateClosed for us. +// There is currently no plan for StateHijacked or hijacking HTTP/2 connections. +func (sc *serverConn) setConnState(state http.ConnState) { + if sc.hs.ConnState != nil { + sc.hs.ConnState(sc.conn, state) + } +} + func (sc *serverConn) vlogf(format string, args ...interface{}) { if VerboseLogs { sc.logf(format, args...) @@ -448,12 +533,55 @@ func (sc *serverConn) logf(format string, args ...interface{}) { } } +// errno returns v's underlying uintptr, else 0. +// +// TODO: remove this helper function once http2 can use build +// tags. See comment in isClosedConnError. +func errno(v error) uintptr { + if rv := reflect.ValueOf(v); rv.Kind() == reflect.Uintptr { + return uintptr(rv.Uint()) + } + return 0 +} + +// isClosedConnError reports whether err is an error from use of a closed +// network connection. +func isClosedConnError(err error) bool { + if err == nil { + return false + } + + // TODO: remove this string search and be more like the Windows + // case below. That might involve modifying the standard library + // to return better error types. + str := err.Error() + if strings.Contains(str, "use of closed network connection") { + return true + } + + // TODO(bradfitz): x/tools/cmd/bundle doesn't really support + // build tags, so I can't make an http2_windows.go file with + // Windows-specific stuff. Fix that and move this, once we + // have a way to bundle this into std's net/http somehow. + if runtime.GOOS == "windows" { + if oe, ok := err.(*net.OpError); ok && oe.Op == "read" { + if se, ok := oe.Err.(*os.SyscallError); ok && se.Syscall == "wsarecv" { + const WSAECONNABORTED = 10053 + const WSAECONNRESET = 10054 + if n := errno(se.Err); n == WSAECONNRESET || n == WSAECONNABORTED { + return true + } + } + } + } + return false +} + func (sc *serverConn) condlogf(err error, format string, args ...interface{}) { if err == nil { return } - str := err.Error() - if err == io.EOF || strings.Contains(str, "use of closed network connection") { + if err == io.EOF || err == io.ErrUnexpectedEOF || isClosedConnError(err) { // Boring, expected errors. sc.vlogf(format, args...) } else { @@ -463,9 +591,11 @@ func (sc *serverConn) condlogf(err error, format string, args ...interface{}) { func (sc *serverConn) onNewHeaderField(f hpack.HeaderField) { sc.serveG.check() - sc.vlogf("got header field %+v", f) + if VerboseLogs { + sc.vlogf("http2: server decoded %v", f) + } switch { - case !validHeader(f.Name): + case !validHeaderFieldValue(f.Value): // f.Name checked _after_ pseudo check, since ':' is invalid sc.req.invalidHeader = true case strings.HasPrefix(f.Name, ":"): if sc.req.sawRegularHeader { @@ -499,16 +629,44 @@ func (sc *serverConn) onNewHeaderField(f hpack.HeaderField) { return } *dst = f.Value - case f.Name == "cookie": - sc.req.sawRegularHeader = true - if s, ok := sc.req.header["Cookie"]; ok && len(s) == 1 { - s[0] = s[0] + "; " + f.Value - } else { - sc.req.header.Add("Cookie", f.Value) - } + case !validHeaderFieldName(f.Name): + sc.req.invalidHeader = true default: sc.req.sawRegularHeader = true sc.req.header.Add(sc.canonicalHeader(f.Name), f.Value) + const headerFieldOverhead = 32 // per spec + sc.req.headerListSize += int64(len(f.Name)) + int64(len(f.Value)) + headerFieldOverhead + if sc.req.headerListSize > int64(sc.maxHeaderListSize()) { + sc.hpackDecoder.SetEmitEnabled(false) + } + } +} + +func (st *stream) onNewTrailerField(f hpack.HeaderField) { + sc := st.sc + sc.serveG.check() + if VerboseLogs { + sc.vlogf("http2: server decoded trailer %v", f) + } + switch { + case strings.HasPrefix(f.Name, ":"): + sc.req.invalidHeader = true + return + case !validHeaderFieldName(f.Name) || !validHeaderFieldValue(f.Value): + sc.req.invalidHeader = true + return + default: + key := sc.canonicalHeader(f.Name) + if st.trailer != nil { + vv := append(st.trailer[key], f.Value) + st.trailer[key] = vv + + // arbitrary; TODO: read spec about header list size limits wrt trailers + const tooBig = 1000 + if len(vv) >= tooBig { + sc.hpackDecoder.SetEmitEnabled(false) + } + } } } @@ -530,41 +688,53 @@ func (sc *serverConn) canonicalHeader(v string) string { return cv } +type readFrameResult struct { + f Frame // valid until readMore is called + err error + + // readMore should be called once the consumer no longer needs or + // retains f. After readMore, f is invalid and more frames can be + // read. + readMore func() +} + // readFrames is the loop that reads incoming frames. +// It takes care to only read one frame at a time, blocking until the +// consumer is done with the frame. // It's run on its own goroutine. func (sc *serverConn) readFrames() { - g := make(gate, 1) + gate := make(gate) for { f, err := sc.framer.ReadFrame() - if err != nil { - sc.readFrameErrCh <- err - close(sc.readFrameCh) + select { + case sc.readFrameCh <- readFrameResult{f, err, gate.Done}: + case <-sc.doneServing: + return + } + select { + case <-gate: + case <-sc.doneServing: + return + } + if terminalReadFrameError(err) { return } - sc.readFrameCh <- frameAndGate{f, g} - // We can't read another frame until this one is - // processed, as the ReadFrame interface doesn't copy - // memory. The Frame accessor methods access the last - // frame's (shared) buffer. So we wait for the - // serve goroutine to tell us it's done: - g.Wait() } } +// frameWriteResult is the message passed from writeFrameAsync to the serve goroutine. +type frameWriteResult struct { + wm frameWriteMsg // what was written (or attempted) + err error // result of the writeFrame call +} + // writeFrameAsync runs in its own goroutine and writes a single frame // and then reports when it's done. // At most one goroutine can be running writeFrameAsync at a time per // serverConn. func (sc *serverConn) writeFrameAsync(wm frameWriteMsg) { err := wm.write.writeFrame(sc) - if ch := wm.done; ch != nil { - select { - case ch <- err: - default: - panic(fmt.Sprintf("unbuffered done channel passed in for type %T", wm.write)) - } - } - sc.wroteFrameCh <- struct{}{} // tickle frame selection scheduler + sc.wroteFrameCh <- frameWriteResult{wm, err} } func (sc *serverConn) closeAllStreamsOnConnClose() { @@ -582,6 +752,7 @@ func (sc *serverConn) stopShutdownTimer() { } func (sc *serverConn) notePanic() { + // Note: this is for serverConn.serve panicking, not http.Handler code. if testHookOnPanicMu != nil { testHookOnPanicMu.Lock() defer testHookOnPanicMu.Unlock() @@ -603,12 +774,15 @@ func (sc *serverConn) serve() { defer sc.stopShutdownTimer() defer close(sc.doneServing) // unblocks handlers trying to send - sc.vlogf("HTTP/2 connection from %v on %p", sc.conn.RemoteAddr(), sc.hs) + if VerboseLogs { + sc.vlogf("http2: server connection from %v on %p", sc.conn.RemoteAddr(), sc.hs) + } sc.writeFrame(frameWriteMsg{ write: writeSettings{ {SettingMaxFrameSize, sc.srv.maxReadFrameSize()}, {SettingMaxConcurrentStreams, sc.advMaxStreams}, + {SettingMaxHeaderListSize, sc.maxHeaderListSize()}, // TODO: more actual settings, notably // SettingInitialWindowSize, but then we also @@ -619,30 +793,32 @@ func (sc *serverConn) serve() { sc.unackedSettings++ if err := sc.readPreface(); err != nil { - sc.condlogf(err, "error reading preface from client %v: %v", sc.conn.RemoteAddr(), err) + sc.condlogf(err, "http2: server: error reading preface from client %v: %v", sc.conn.RemoteAddr(), err) return } + // Now that we've got the preface, get us out of the + // "StateNew" state. We can't go directly to idle, though. + // Active means we read some data and anticipate a request. We'll + // do another Active when we get a HEADERS frame. + sc.setConnState(http.StateActive) + sc.setConnState(http.StateIdle) go sc.readFrames() // closed by defer sc.conn.Close above settingsTimer := time.NewTimer(firstSettingsTimeout) + loopNum := 0 for { + loopNum++ select { case wm := <-sc.wantWriteFrameCh: sc.writeFrame(wm) - case <-sc.wroteFrameCh: - if sc.writingFrame != true { - panic("internal error: expected to be already writing a frame") - } - sc.writingFrame = false - sc.scheduleFrameWrite() - case fg, ok := <-sc.readFrameCh: - if !ok { - sc.readFrameCh = nil - } - if !sc.processFrameFromReader(fg, ok) { + case res := <-sc.wroteFrameCh: + sc.wroteFrame(res) + case res := <-sc.readFrameCh: + if !sc.processFrameFromReader(res) { return } + res.readMore() if settingsTimer.C != nil { settingsTimer.Stop() settingsTimer.C = nil @@ -656,7 +832,7 @@ func (sc *serverConn) serve() { sc.vlogf("GOAWAY close timer fired; closing conn from %v", sc.conn.RemoteAddr()) return case fn := <-sc.testHookCh: - fn() + fn(loopNum) } } } @@ -683,38 +859,62 @@ func (sc *serverConn) readPreface() error { return errors.New("timeout waiting for client preface") case err := <-errc: if err == nil { - sc.vlogf("client %v said hello", sc.conn.RemoteAddr()) + if VerboseLogs { + sc.vlogf("http2: server: client %v said hello", sc.conn.RemoteAddr()) + } } return err } } -// writeDataFromHandler writes the data described in req to stream.id. -// -// The provided ch is used to avoid allocating new channels for each -// write operation. It's expected that the caller reuses writeData and ch -// over time. -// -// The flow control currently happens in the Handler where it waits -// for 1 or more bytes to be available to then write here. So at this -// point we know that we have flow control. But this might have to -// change when priority is implemented, so the serve goroutine knows -// the total amount of bytes waiting to be sent and can can have more -// scheduling decisions available. -func (sc *serverConn) writeDataFromHandler(stream *stream, writeData *writeData, ch chan error) error { - sc.writeFrameFromHandler(frameWriteMsg{ - write: writeData, +var errChanPool = sync.Pool{ + New: func() interface{} { return make(chan error, 1) }, +} + +var writeDataPool = sync.Pool{ + New: func() interface{} { return new(writeData) }, +} + +// writeDataFromHandler writes DATA response frames from a handler on +// the given stream. +func (sc *serverConn) writeDataFromHandler(stream *stream, data []byte, endStream bool) error { + ch := errChanPool.Get().(chan error) + writeArg := writeDataPool.Get().(*writeData) + *writeArg = writeData{stream.id, data, endStream} + err := sc.writeFrameFromHandler(frameWriteMsg{ + write: writeArg, stream: stream, done: ch, }) - select { - case err := <-ch: + if err != nil { return err + } + var frameWriteDone bool // the frame write is done (successfully or not) + select { + case err = <-ch: + frameWriteDone = true case <-sc.doneServing: return errClientDisconnected case <-stream.cw: - return errStreamBroken + // If both ch and stream.cw were ready (as might + // happen on the final Write after an http.Handler + // ends), prefer the write result. Otherwise this + // might just be us successfully closing the stream. + // The writeFrameAsync and serve goroutines guarantee + // that the ch send will happen before the stream.cw + // close. + select { + case err = <-ch: + frameWriteDone = true + default: + return errStreamClosed + } } + errChanPool.Put(ch) + if frameWriteDone { + writeDataPool.Put(writeArg) + } + return err } // writeFrameFromHandler sends wm to sc.wantWriteFrameCh, but aborts @@ -724,12 +924,15 @@ func (sc *serverConn) writeDataFromHandler(stream *stream, writeData *writeData, // deadlock writing to sc.wantWriteFrameCh (which is only mildly // buffered and is read by serve itself). If you're on the serve // goroutine, call writeFrame instead. -func (sc *serverConn) writeFrameFromHandler(wm frameWriteMsg) { +func (sc *serverConn) writeFrameFromHandler(wm frameWriteMsg) error { sc.serveG.checkNotOn() // NOT select { case sc.wantWriteFrameCh <- wm: + return nil case <-sc.doneServing: + // Serve loop is gone. // Client has closed their connection to the server. + return errClientDisconnected } } @@ -755,7 +958,6 @@ func (sc *serverConn) startFrameWrite(wm frameWriteMsg) { if sc.writingFrame { panic("internal error: can only be writing one frame at a time") } - sc.writingFrame = true st := wm.stream if st != nil { @@ -764,16 +966,53 @@ func (sc *serverConn) startFrameWrite(wm frameWriteMsg) { panic("internal error: attempt to send frame on half-closed-local stream") case stateClosed: if st.sentReset || st.gotReset { - // Skip this frame. But fake the frame write to reschedule: - sc.wroteFrameCh <- struct{}{} + // Skip this frame. + sc.scheduleFrameWrite() return } panic(fmt.Sprintf("internal error: attempt to send a write %v on a closed stream", wm)) } } + sc.writingFrame = true sc.needsFrameFlush = true - if endsStream(wm.write) { + go sc.writeFrameAsync(wm) +} + +// errHandlerPanicked is the error given to any callers blocked in a read from +// Request.Body when the main goroutine panics. Since most handlers read in the +// the main ServeHTTP goroutine, this will show up rarely. +var errHandlerPanicked = errors.New("http2: handler panicked") + +// wroteFrame is called on the serve goroutine with the result of +// whatever happened on writeFrameAsync. +func (sc *serverConn) wroteFrame(res frameWriteResult) { + sc.serveG.check() + if !sc.writingFrame { + panic("internal error: expected to be already writing a frame") + } + sc.writingFrame = false + + wm := res.wm + st := wm.stream + + closeStream := endsStream(wm.write) + + if _, ok := wm.write.(handlerPanicRST); ok { + sc.closeStream(st, errHandlerPanicked) + } + + // Reply (if requested) to the blocked ServeHTTP goroutine. + if ch := wm.done; ch != nil { + select { + case ch <- res.err: + default: + panic(fmt.Sprintf("unbuffered done channel passed in for type %T", wm.write)) + } + } + wm.write = nil // prevent use (assume it's tainted after wm.done send) + + if closeStream { if st == nil { panic("internal error: expecting non-nil stream") } @@ -791,10 +1030,11 @@ func (sc *serverConn) startFrameWrite(wm frameWriteMsg) { errCancel := StreamError{st.id, ErrCodeCancel} sc.resetStream(errCancel) case stateHalfClosedRemote: - sc.closeStream(st, nil) + sc.closeStream(st, errHandlerComplete) } } - go sc.writeFrameAsync(wm) + + sc.scheduleFrameWrite() } // scheduleFrameWrite tickles the frame writing scheduler. @@ -874,32 +1114,18 @@ func (sc *serverConn) resetStream(se StreamError) { } } -// curHeaderStreamID returns the stream ID of the header block we're -// currently in the middle of reading. If this returns non-zero, the -// next frame must be a CONTINUATION with this stream id. -func (sc *serverConn) curHeaderStreamID() uint32 { - sc.serveG.check() - st := sc.req.stream - if st == nil { - return 0 - } - return st.id -} - // processFrameFromReader processes the serve loop's read from readFrameCh from the // frame-reading goroutine. // processFrameFromReader returns whether the connection should be kept open. -func (sc *serverConn) processFrameFromReader(fg frameAndGate, fgValid bool) bool { +func (sc *serverConn) processFrameFromReader(res readFrameResult) bool { sc.serveG.check() - var clientGone bool - var err error - if !fgValid { - err = <-sc.readFrameErrCh + err := res.err + if err != nil { if err == ErrFrameTooLarge { sc.goAway(ErrCodeFrameSize) return true // goAway will close the loop } - clientGone = err == io.EOF || strings.Contains(err.Error(), "use of closed network connection") + clientGone := err == io.EOF || err == io.ErrUnexpectedEOF || isClosedConnError(err) if clientGone { // TODO: could we also get into this state if // the peer does a half close @@ -911,13 +1137,12 @@ func (sc *serverConn) processFrameFromReader(fg frameAndGate, fgValid bool) bool // just for testing we could have a non-TLS mode. return false } - } - - if fgValid { - f := fg.f - sc.vlogf("got %v: %#v", f.Header(), f) + } else { + f := res.f + if VerboseLogs { + sc.vlogf("http2: server read frame %v", summarizeFrame(f)) + } err = sc.processFrame(f) - fg.g.Done() // unblock the readFrames goroutine if err == nil { return true } @@ -931,17 +1156,17 @@ func (sc *serverConn) processFrameFromReader(fg frameAndGate, fgValid bool) bool sc.goAway(ErrCodeFlowControl) return true case ConnectionError: - sc.logf("%v: %v", sc.conn.RemoteAddr(), ev) + sc.logf("http2: server connection error from %v: %v", sc.conn.RemoteAddr(), ev) sc.goAway(ErrCode(ev)) return true // goAway will handle shutdown default: - if !fgValid { - sc.logf("disconnecting; error reading frame from client %s: %v", sc.conn.RemoteAddr(), err) + if res.err != nil { + sc.vlogf("http2: server closing client connection; error reading frame from client %s: %v", sc.conn.RemoteAddr(), err) } else { - sc.logf("disconnection due to other error: %v", err) + sc.logf("http2: server closing client connection: %v", err) } + return false } - return false } func (sc *serverConn) processFrame(f Frame) error { @@ -955,14 +1180,6 @@ func (sc *serverConn) processFrame(f Frame) error { sc.sawFirstSettings = true } - if s := sc.curHeaderStreamID(); s != 0 { - if cf, ok := f.(*ContinuationFrame); !ok { - return ConnectionError(ErrCodeProtocol) - } else if cf.Header().StreamID != s { - return ConnectionError(ErrCodeProtocol) - } - } - switch f := f.(type) { case *SettingsFrame: return sc.processSettings(f) @@ -985,14 +1202,14 @@ func (sc *serverConn) processFrame(f Frame) error { // frame as a connection error (Section 5.4.1) of type PROTOCOL_ERROR. return ConnectionError(ErrCodeProtocol) default: - log.Printf("Ignoring frame: %v", f.Header()) + sc.vlogf("http2: server ignoring frame: %v", f.Header()) return nil } } func (sc *serverConn) processPing(f *PingFrame) error { sc.serveG.check() - if f.Flags.Has(FlagSettingsAck) { + if f.IsAck() { // 6.7 PING: " An endpoint MUST NOT respond to PING frames // containing this flag." return nil @@ -1060,9 +1277,12 @@ func (sc *serverConn) closeStream(st *stream, err error) { } st.state = stateClosed sc.curOpenStreams-- + if sc.curOpenStreams == 0 { + sc.setConnState(http.StateIdle) + } delete(sc.streams, st.id) if p := st.body; p != nil { - p.Close(err) + p.CloseWithError(err) } st.cw.Close() // signals Handler's CloseNotifier, unblocks writes, etc sc.writeSched.forgetStream(st.id) @@ -1093,7 +1313,9 @@ func (sc *serverConn) processSetting(s Setting) error { if err := s.Valid(); err != nil { return err } - sc.vlogf("processing setting %v", s) + if VerboseLogs { + sc.vlogf("http2: server processing setting %v", s) + } switch s.ID { case SettingHeaderTableSize: sc.headerTableSize = s.Val @@ -1107,11 +1329,14 @@ func (sc *serverConn) processSetting(s Setting) error { case SettingMaxFrameSize: sc.writeSched.maxFrameSize = s.Val case SettingMaxHeaderListSize: - sc.maxHeaderListSize = s.Val + sc.peerMaxHeaderListSize = s.Val default: // Unknown setting: "An endpoint that receives a SETTINGS // frame with any unknown or unsupported identifier MUST // ignore that setting." + if VerboseLogs { + sc.vlogf("http2: server ignoring unknown setting %v", s) + } } return nil } @@ -1151,7 +1376,7 @@ func (sc *serverConn) processData(f *DataFrame) error { // with a stream error (Section 5.4.2) of type STREAM_CLOSED." id := f.Header().StreamID st, ok := sc.streams[id] - if !ok || st.state != stateOpen { + if !ok || st.state != stateOpen || st.gotTrailerHeader { // This includes sending a RST_STREAM if the stream is // in stateHalfClosedLocal (which currently means that // the http.Handler returned, so it's done reading & @@ -1166,7 +1391,7 @@ func (sc *serverConn) processData(f *DataFrame) error { // Sender sending more than they'd declared? if st.declBodyBytes != -1 && st.bodyBytes+int64(len(data)) > st.declBodyBytes { - st.body.Close(fmt.Errorf("sender tried to send more than declared Content-Length of %d bytes", st.declBodyBytes)) + st.body.CloseWithError(fmt.Errorf("sender tried to send more than declared Content-Length of %d bytes", st.declBodyBytes)) return StreamError{id, ErrCodeStreamClosed} } if len(data) > 0 { @@ -1185,17 +1410,38 @@ func (sc *serverConn) processData(f *DataFrame) error { st.bodyBytes += int64(len(data)) } if f.StreamEnded() { - if st.declBodyBytes != -1 && st.declBodyBytes != st.bodyBytes { - st.body.Close(fmt.Errorf("request declared a Content-Length of %d but only wrote %d bytes", - st.declBodyBytes, st.bodyBytes)) - } else { - st.body.Close(io.EOF) - } - st.state = stateHalfClosedRemote + st.endStream() } return nil } +// endStream closes a Request.Body's pipe. It is called when a DATA +// frame says a request body is over (or after trailers). +func (st *stream) endStream() { + sc := st.sc + sc.serveG.check() + + if st.declBodyBytes != -1 && st.declBodyBytes != st.bodyBytes { + st.body.CloseWithError(fmt.Errorf("request declared a Content-Length of %d but only wrote %d bytes", + st.declBodyBytes, st.bodyBytes)) + } else { + st.body.closeWithErrorAndCode(io.EOF, st.copyTrailersToHandlerRequest) + st.body.CloseWithError(io.EOF) + } + st.state = stateHalfClosedRemote +} + +// copyTrailersToHandlerRequest is run in the Handler's goroutine in +// its Request.Body.Read just before it gets io.EOF. +func (st *stream) copyTrailersToHandlerRequest() { + for k, vv := range st.trailer { + if _, ok := st.reqTrailer[k]; ok { + // Only copy it over it was pre-declared. + st.reqTrailer[k] = vv + } + } +} + func (sc *serverConn) processHeaders(f *HeadersFrame) error { sc.serveG.check() id := f.Header().StreamID @@ -1204,20 +1450,36 @@ func (sc *serverConn) processHeaders(f *HeadersFrame) error { return nil } // http://http2.github.io/http2-spec/#rfc.section.5.1.1 - if id%2 != 1 || id <= sc.maxStreamID || sc.req.stream != nil { - // Streams initiated by a client MUST use odd-numbered - // stream identifiers. [...] The identifier of a newly - // established stream MUST be numerically greater than all - // streams that the initiating endpoint has opened or - // reserved. [...] An endpoint that receives an unexpected - // stream identifier MUST respond with a connection error - // (Section 5.4.1) of type PROTOCOL_ERROR. + // Streams initiated by a client MUST use odd-numbered stream + // identifiers. [...] An endpoint that receives an unexpected + // stream identifier MUST respond with a connection error + // (Section 5.4.1) of type PROTOCOL_ERROR. + if id%2 != 1 { return ConnectionError(ErrCodeProtocol) } + // A HEADERS frame can be used to create a new stream or + // send a trailer for an open one. If we already have a stream + // open, let it process its own HEADERS frame (trailers at this + // point, if it's valid). + st := sc.streams[f.Header().StreamID] + if st != nil { + return st.processTrailerHeaders(f) + } + + // [...] The identifier of a newly established stream MUST be + // numerically greater than all streams that the initiating + // endpoint has opened or reserved. [...] An endpoint that + // receives an unexpected stream identifier MUST respond with + // a connection error (Section 5.4.1) of type PROTOCOL_ERROR. + if id <= sc.maxStreamID || sc.req.stream != nil { + return ConnectionError(ErrCodeProtocol) + } + if id > sc.maxStreamID { sc.maxStreamID = id } - st := &stream{ + st = &stream{ + sc: sc, id: id, state: stateOpen, } @@ -1236,18 +1498,37 @@ func (sc *serverConn) processHeaders(f *HeadersFrame) error { adjustStreamPriority(sc.streams, st.id, f.Priority) } sc.curOpenStreams++ + if sc.curOpenStreams == 1 { + sc.setConnState(http.StateActive) + } sc.req = requestParam{ stream: st, header: make(http.Header), } + sc.hpackDecoder.SetEmitFunc(sc.onNewHeaderField) + sc.hpackDecoder.SetEmitEnabled(true) return sc.processHeaderBlockFragment(st, f.HeaderBlockFragment(), f.HeadersEnded()) } +func (st *stream) processTrailerHeaders(f *HeadersFrame) error { + sc := st.sc + sc.serveG.check() + if st.gotTrailerHeader { + return ConnectionError(ErrCodeProtocol) + } + st.gotTrailerHeader = true + if !f.StreamEnded() { + return StreamError{st.id, ErrCodeProtocol} + } + sc.resetPendingRequest() // we use invalidHeader from it for trailers + return st.processTrailerHeaderBlockFragment(f.HeaderBlockFragment(), f.HeadersEnded()) +} + func (sc *serverConn) processContinuation(f *ContinuationFrame) error { sc.serveG.check() st := sc.streams[f.Header().StreamID] - if st == nil || sc.curHeaderStreamID() != st.id { - return ConnectionError(ErrCodeProtocol) + if st.gotTrailerHeader { + return st.processTrailerHeaderBlockFragment(f.HeaderBlockFragment(), f.HeadersEnded()) } return sc.processHeaderBlockFragment(st, f.HeaderBlockFragment(), f.HeadersEnded()) } @@ -1255,15 +1536,13 @@ func (sc *serverConn) processContinuation(f *ContinuationFrame) error { func (sc *serverConn) processHeaderBlockFragment(st *stream, frag []byte, end bool) error { sc.serveG.check() if _, err := sc.hpackDecoder.Write(frag); err != nil { - // TODO: convert to stream error I assume? - return err + return ConnectionError(ErrCodeCompression) } if !end { return nil } if err := sc.hpackDecoder.Close(); err != nil { - // TODO: convert to stream error I assume? - return err + return ConnectionError(ErrCodeCompression) } defer sc.resetPendingRequest() if sc.curOpenStreams > sc.advMaxStreams { @@ -1289,9 +1568,44 @@ func (sc *serverConn) processHeaderBlockFragment(st *stream, frag []byte, end bo if err != nil { return err } + st.reqTrailer = req.Trailer + if st.reqTrailer != nil { + st.trailer = make(http.Header) + } st.body = req.Body.(*requestBody).pipe // may be nil st.declBodyBytes = req.ContentLength - go sc.runHandler(rw, req) + + handler := sc.handler.ServeHTTP + if !sc.hpackDecoder.EmitEnabled() { + // Their header list was too long. Send a 431 error. + handler = handleHeaderListTooLong + } + + go sc.runHandler(rw, req, handler) + return nil +} + +func (st *stream) processTrailerHeaderBlockFragment(frag []byte, end bool) error { + sc := st.sc + sc.serveG.check() + sc.hpackDecoder.SetEmitFunc(st.onNewTrailerField) + if _, err := sc.hpackDecoder.Write(frag); err != nil { + return ConnectionError(ErrCodeCompression) + } + if !end { + return nil + } + + rp := &sc.req + if rp.invalidHeader { + return StreamError{rp.stream.id, ErrCodeProtocol} + } + + err := sc.hpackDecoder.Close() + st.endStream() + if err != nil { + return ConnectionError(ErrCodeCompression) + } return nil } @@ -1347,7 +1661,17 @@ func (sc *serverConn) resetPendingRequest() { func (sc *serverConn) newWriterAndRequest() (*responseWriter, *http.Request, error) { sc.serveG.check() rp := &sc.req - if rp.invalidHeader || rp.method == "" || rp.path == "" || + + if rp.invalidHeader { + return nil, nil, StreamError{rp.stream.id, ErrCodeProtocol} + } + + isConnect := rp.method == "CONNECT" + if isConnect { + if rp.path != "" || rp.scheme != "" || rp.authority == "" { + return nil, nil, StreamError{rp.stream.id, ErrCodeProtocol} + } + } else if rp.method == "" || rp.path == "" || (rp.scheme != "https" && rp.scheme != "http") { // See 8.1.2.6 Malformed Requests and Responses: // @@ -1361,7 +1685,14 @@ func (sc *serverConn) newWriterAndRequest() (*responseWriter, *http.Request, err // pseudo-header fields" return nil, nil, StreamError{rp.stream.id, ErrCodeProtocol} } + + bodyOpen := rp.stream.state == stateOpen + if rp.method == "HEAD" && bodyOpen { + // HEAD requests can't have bodies + return nil, nil, StreamError{rp.stream.id, ErrCodeProtocol} + } var tlsState *tls.ConnectionState // nil if not scheme https + if rp.scheme == "https" { tlsState = sc.tlsState } @@ -1373,36 +1704,66 @@ func (sc *serverConn) newWriterAndRequest() (*responseWriter, *http.Request, err if needsContinue { rp.header.Del("Expect") } - bodyOpen := rp.stream.state == stateOpen + // Merge Cookie headers into one "; "-delimited value. + if cookies := rp.header["Cookie"]; len(cookies) > 1 { + rp.header.Set("Cookie", strings.Join(cookies, "; ")) + } + + // Setup Trailers + var trailer http.Header + for _, v := range rp.header["Trailer"] { + for _, key := range strings.Split(v, ",") { + key = http.CanonicalHeaderKey(strings.TrimSpace(key)) + switch key { + case "Transfer-Encoding", "Trailer", "Content-Length": + // Bogus. (copy of http1 rules) + // Ignore. + default: + if trailer == nil { + trailer = make(http.Header) + } + trailer[key] = nil + } + } + } + delete(rp.header, "Trailer") + body := &requestBody{ conn: sc, stream: rp.stream, needsContinue: needsContinue, } - // TODO: handle asterisk '*' requests + test - url, err := url.ParseRequestURI(rp.path) - if err != nil { - // TODO: find the right error code? - return nil, nil, StreamError{rp.stream.id, ErrCodeProtocol} + var url_ *url.URL + var requestURI string + if isConnect { + url_ = &url.URL{Host: rp.authority} + requestURI = rp.authority // mimic HTTP/1 server behavior + } else { + var err error + url_, err = url.ParseRequestURI(rp.path) + if err != nil { + return nil, nil, StreamError{rp.stream.id, ErrCodeProtocol} + } + requestURI = rp.path } req := &http.Request{ Method: rp.method, - URL: url, + URL: url_, RemoteAddr: sc.remoteAddrStr, Header: rp.header, - RequestURI: rp.path, + RequestURI: requestURI, Proto: "HTTP/2.0", ProtoMajor: 2, ProtoMinor: 0, TLS: tlsState, Host: authority, Body: body, + Trailer: trailer, } if bodyOpen { body.pipe = &pipe{ - b: buffer{buf: make([]byte, initialWindowSize)}, // TODO: share/remove XXX + b: &fixedBuffer{buf: make([]byte, initialWindowSize)}, // TODO: garbage } - body.pipe.c.L = &body.pipe.m if vv, ok := rp.header["Content-Length"]; ok { req.ContentLength, _ = strconv.ParseInt(vv[0], 10, 64) @@ -1420,22 +1781,47 @@ func (sc *serverConn) newWriterAndRequest() (*responseWriter, *http.Request, err rws.stream = rp.stream rws.req = req rws.body = body - rws.frameWriteCh = make(chan error, 1) rw := &responseWriter{rws: rws} return rw, req, nil } // Run on its own goroutine. -func (sc *serverConn) runHandler(rw *responseWriter, req *http.Request) { - defer rw.handlerDone() - // TODO: catch panics like net/http.Server - sc.handler.ServeHTTP(rw, req) +func (sc *serverConn) runHandler(rw *responseWriter, req *http.Request, handler func(http.ResponseWriter, *http.Request)) { + didPanic := true + defer func() { + if didPanic { + e := recover() + // Same as net/http: + const size = 64 << 10 + buf := make([]byte, size) + buf = buf[:runtime.Stack(buf, false)] + sc.writeFrameFromHandler(frameWriteMsg{ + write: handlerPanicRST{rw.rws.stream.id}, + stream: rw.rws.stream, + }) + sc.logf("http2: panic serving %v: %v\n%s", sc.conn.RemoteAddr(), e, buf) + return + } + rw.handlerDone() + }() + handler(rw, req) + didPanic = false +} + +func handleHeaderListTooLong(w http.ResponseWriter, r *http.Request) { + // 10.5.1 Limits on Header Block Size: + // .. "A server that receives a larger header block than it is + // willing to handle can send an HTTP 431 (Request Header Fields Too + // Large) status code" + const statusRequestHeaderFieldsTooLarge = 431 // only in Go 1.6+ + w.WriteHeader(statusRequestHeaderFieldsTooLarge) + io.WriteString(w, "

HTTP Error 431

Request Header Field(s) Too Large

") } // called from handler goroutines. // h may be nil. -func (sc *serverConn) writeHeaders(st *stream, headerData *writeResHeaders, tempCh chan error) { +func (sc *serverConn) writeHeaders(st *stream, headerData *writeResHeaders) error { sc.serveG.checkNotOn() // NOT on var errc chan error if headerData.h != nil { @@ -1443,22 +1829,27 @@ func (sc *serverConn) writeHeaders(st *stream, headerData *writeResHeaders, temp // waiting for this frame to be written, so an http.Flush mid-handler // writes out the correct value of keys, before a handler later potentially // mutates it. - errc = tempCh + errc = errChanPool.Get().(chan error) } - sc.writeFrameFromHandler(frameWriteMsg{ + if err := sc.writeFrameFromHandler(frameWriteMsg{ write: headerData, stream: st, done: errc, - }) + }); err != nil { + return err + } if errc != nil { select { - case <-errc: - // Ignore. Just for synchronization. - // Any error will be handled in the writing goroutine. + case err := <-errc: + errChanPool.Put(errc) + return err case <-sc.doneServing: - // Client has closed the connection. + return errClientDisconnected + case <-st.cw: + return errStreamClosed } } + return nil } // called from handler goroutines. @@ -1481,7 +1872,10 @@ type bodyReadMsg struct { // and schedules flow control tokens to be sent. func (sc *serverConn) noteBodyReadFromHandler(st *stream, n int) { sc.serveG.checkNotOn() // NOT on - sc.bodyReadCh <- bodyReadMsg{st, n} + select { + case sc.bodyReadCh <- bodyReadMsg{st, n}: + case <-sc.doneServing: + } } func (sc *serverConn) noteBodyRead(st *stream, n int) { @@ -1548,7 +1942,7 @@ type requestBody struct { func (b *requestBody) Close() error { if b.pipe != nil { - b.pipe.Close(errClosedBody) + b.pipe.CloseWithError(errClosedBody) } b.closed = true return nil @@ -1599,12 +1993,14 @@ type responseWriterState struct { // mutated by http.Handler goroutine: handlerHeader http.Header // nil until called snapHeader http.Header // snapshot of handlerHeader at WriteHeader time + trailers []string // set in writeChunk status int // status code passed to WriteHeader wroteHeader bool // WriteHeader called (explicitly or implicitly). Not necessarily sent to user yet. sentHeader bool // have we sent the header frame? handlerDone bool // handler has finished - curWrite writeData - frameWriteCh chan error // re-used whenever we need to block on a frame being written + + sentContentLen int64 // non-zero if handler set a Content-Length header + wroteBytes int64 closeNotifierMu sync.Mutex // guards closeNotifierCh closeNotifierCh chan bool // nil until first used @@ -1614,6 +2010,23 @@ type chunkWriter struct{ rws *responseWriterState } func (cw chunkWriter) Write(p []byte) (n int, err error) { return cw.rws.writeChunk(p) } +func (rws *responseWriterState) hasTrailers() bool { return len(rws.trailers) != 0 } + +// declareTrailer is called for each Trailer header when the +// response header is written. It notes that a header will need to be +// written in the trailers at the end of the response. +func (rws *responseWriterState) declareTrailer(k string) { + k = http.CanonicalHeaderKey(k) + switch k { + case "Transfer-Encoding", "Content-Length", "Trailer": + // Forbidden by RFC 2616 14.40. + return + } + if !strSliceContains(rws.trailers, k) { + rws.trailers = append(rws.trailers, k) + } +} + // writeChunk writes chunks from the bufio.Writer. But because // bufio.Writer may bypass its chunking, sometimes p may be // arbitrarily large. @@ -1624,41 +2037,132 @@ func (rws *responseWriterState) writeChunk(p []byte) (n int, err error) { if !rws.wroteHeader { rws.writeHeader(200) } + + isHeadResp := rws.req.Method == "HEAD" if !rws.sentHeader { rws.sentHeader = true - var ctype, clen string // implicit ones, if we can calculate it - if rws.handlerDone && rws.snapHeader.Get("Content-Length") == "" { + var ctype, clen string + if clen = rws.snapHeader.Get("Content-Length"); clen != "" { + rws.snapHeader.Del("Content-Length") + clen64, err := strconv.ParseInt(clen, 10, 64) + if err == nil && clen64 >= 0 { + rws.sentContentLen = clen64 + } else { + clen = "" + } + } + if clen == "" && rws.handlerDone && bodyAllowedForStatus(rws.status) && (len(p) > 0 || !isHeadResp) { clen = strconv.Itoa(len(p)) } - if rws.snapHeader.Get("Content-Type") == "" { + _, hasContentType := rws.snapHeader["Content-Type"] + if !hasContentType && bodyAllowedForStatus(rws.status) { ctype = http.DetectContentType(p) } - endStream := rws.handlerDone && len(p) == 0 - rws.conn.writeHeaders(rws.stream, &writeResHeaders{ + var date string + if _, ok := rws.snapHeader["Date"]; !ok { + // TODO(bradfitz): be faster here, like net/http? measure. + date = time.Now().UTC().Format(http.TimeFormat) + } + + for _, v := range rws.snapHeader["Trailer"] { + foreachHeaderElement(v, rws.declareTrailer) + } + + endStream := (rws.handlerDone && !rws.hasTrailers() && len(p) == 0) || isHeadResp + err = rws.conn.writeHeaders(rws.stream, &writeResHeaders{ streamID: rws.stream.id, httpResCode: rws.status, h: rws.snapHeader, endStream: endStream, contentType: ctype, contentLength: clen, - }, rws.frameWriteCh) + date: date, + }) + if err != nil { + return 0, err + } if endStream { return 0, nil } } + if isHeadResp { + return len(p), nil + } if len(p) == 0 && !rws.handlerDone { return 0, nil } - curWrite := &rws.curWrite - curWrite.streamID = rws.stream.id - curWrite.p = p - curWrite.endStream = rws.handlerDone - if err := rws.conn.writeDataFromHandler(rws.stream, curWrite, rws.frameWriteCh); err != nil { - return 0, err + + if rws.handlerDone { + rws.promoteUndeclaredTrailers() + } + + endStream := rws.handlerDone && !rws.hasTrailers() + if len(p) > 0 || endStream { + // only send a 0 byte DATA frame if we're ending the stream. + if err := rws.conn.writeDataFromHandler(rws.stream, p, endStream); err != nil { + return 0, err + } + } + + if rws.handlerDone && rws.hasTrailers() { + err = rws.conn.writeHeaders(rws.stream, &writeResHeaders{ + streamID: rws.stream.id, + h: rws.handlerHeader, + trailers: rws.trailers, + endStream: true, + }) + return len(p), err } return len(p), nil } +// TrailerPrefix is a magic prefix for ResponseWriter.Header map keys +// that, if present, signals that the map entry is actually for +// the response trailers, and not the response headers. The prefix +// is stripped after the ServeHTTP call finishes and the values are +// sent in the trailers. +// +// This mechanism is intended only for trailers that are not known +// prior to the headers being written. If the set of trailers is fixed +// or known before the header is written, the normal Go trailers mechanism +// is preferred: +// https://golang.org/pkg/net/http/#ResponseWriter +// https://golang.org/pkg/net/http/#example_ResponseWriter_trailers +const TrailerPrefix = "Trailer:" + +// promoteUndeclaredTrailers permits http.Handlers to set trailers +// after the header has already been flushed. Because the Go +// ResponseWriter interface has no way to set Trailers (only the +// Header), and because we didn't want to expand the ResponseWriter +// interface, and because nobody used trailers, and because RFC 2616 +// says you SHOULD (but not must) predeclare any trailers in the +// header, the official ResponseWriter rules said trailers in Go must +// be predeclared, and then we reuse the same ResponseWriter.Header() +// map to mean both Headers and Trailers. When it's time to write the +// Trailers, we pick out the fields of Headers that were declared as +// trailers. That worked for a while, until we found the first major +// user of Trailers in the wild: gRPC (using them only over http2), +// and gRPC libraries permit setting trailers mid-stream without +// predeclarnig them. So: change of plans. We still permit the old +// way, but we also permit this hack: if a Header() key begins with +// "Trailer:", the suffix of that key is a Trailer. Because ':' is an +// invalid token byte anyway, there is no ambiguity. (And it's already +// filtered out) It's mildly hacky, but not terrible. +// +// This method runs after the Handler is done and promotes any Header +// fields to be trailers. +func (rws *responseWriterState) promoteUndeclaredTrailers() { + for k, vv := range rws.handlerHeader { + if !strings.HasPrefix(k, TrailerPrefix) { + continue + } + trailerKey := strings.TrimPrefix(k, TrailerPrefix) + rws.declareTrailer(trailerKey) + rws.handlerHeader[http.CanonicalHeaderKey(trailerKey)] = vv + } + sort.Strings(rws.trailers) +} + func (w *responseWriter) Flush() { rws := w.rws if rws == nil { @@ -1761,6 +2265,15 @@ func (w *responseWriter) write(lenData int, dataB []byte, dataS string) (n int, if !rws.wroteHeader { w.WriteHeader(200) } + if !bodyAllowedForStatus(rws.status) { + return 0, http.ErrBodyNotAllowed + } + rws.wroteBytes += int64(len(dataB)) + int64(len(dataS)) // only one can be set + if rws.sentContentLen != 0 && rws.wroteBytes > rws.sentContentLen { + // TODO: send a RST_STREAM + return 0, errors.New("http2: handler wrote more than declared Content-Length") + } + if dataB != nil { return rws.bw.Write(dataB) } else { @@ -1770,11 +2283,26 @@ func (w *responseWriter) write(lenData int, dataB []byte, dataS string) (n int, func (w *responseWriter) handlerDone() { rws := w.rws - if rws == nil { - panic("handlerDone called twice") - } rws.handlerDone = true w.Flush() w.rws = nil responseWriterStatePool.Put(rws) } + +// foreachHeaderElement splits v according to the "#rule" construction +// in RFC 2616 section 2.1 and calls fn for each non-empty element. +func foreachHeaderElement(v string, fn func(string)) { + v = textproto.TrimString(v) + if v == "" { + return + } + if !strings.Contains(v, ",") { + fn(v) + return + } + for _, f := range strings.Split(v, ",") { + if f = textproto.TrimString(f); f != "" { + fn(f) + } + } +} diff --git a/vendor/golang.org/x/net/http2/transport.go b/vendor/golang.org/x/net/http2/transport.go index 73f358eefed..c3a1bdb8cb7 100644 --- a/vendor/golang.org/x/net/http2/transport.go +++ b/vendor/golang.org/x/net/http2/transport.go @@ -1,55 +1,156 @@ -// Copyright 2015 The Go Authors. -// See https://go.googlesource.com/go/+/master/CONTRIBUTORS -// Licensed under the same terms as Go itself: -// https://go.googlesource.com/go/+/master/LICENSE +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Transport code. package http2 import ( "bufio" "bytes" + "compress/gzip" "crypto/tls" "errors" "fmt" "io" + "io/ioutil" "log" "net" "net/http" + "sort" "strconv" "strings" "sync" + "time" "golang.org/x/net/http2/hpack" ) +const ( + // transportDefaultConnFlow is how many connection-level flow control + // tokens we give the server at start-up, past the default 64k. + transportDefaultConnFlow = 1 << 30 + + // transportDefaultStreamFlow is how many stream-level flow + // control tokens we announce to the peer, and how many bytes + // we buffer per stream. + transportDefaultStreamFlow = 4 << 20 + + // transportDefaultStreamMinRefresh is the minimum number of bytes we'll send + // a stream-level WINDOW_UPDATE for at a time. + transportDefaultStreamMinRefresh = 4 << 10 + + defaultUserAgent = "Go-http-client/2.0" +) + +// Transport is an HTTP/2 Transport. +// +// A Transport internally caches connections to servers. It is safe +// for concurrent use by multiple goroutines. type Transport struct { - Fallback http.RoundTripper + // DialTLS specifies an optional dial function for creating + // TLS connections for requests. + // + // If DialTLS is nil, tls.Dial is used. + // + // If the returned net.Conn has a ConnectionState method like tls.Conn, + // it will be used to set http.Response.TLS. + DialTLS func(network, addr string, cfg *tls.Config) (net.Conn, error) - // TODO: remove this and make more general with a TLS dial hook, like http - InsecureTLSDial bool + // TLSClientConfig specifies the TLS configuration to use with + // tls.Client. If nil, the default configuration is used. + TLSClientConfig *tls.Config - connMu sync.Mutex - conns map[string][]*clientConn // key is host:port + // ConnPool optionally specifies an alternate connection pool to use. + // If nil, the default is used. + ConnPool ClientConnPool + + // DisableCompression, if true, prevents the Transport from + // requesting compression with an "Accept-Encoding: gzip" + // request header when the Request contains no existing + // Accept-Encoding value. If the Transport requests gzip on + // its own and gets a gzipped response, it's transparently + // decoded in the Response.Body. However, if the user + // explicitly requested gzip it is not automatically + // uncompressed. + DisableCompression bool + + // MaxHeaderListSize is the http2 SETTINGS_MAX_HEADER_LIST_SIZE to + // send in the initial settings frame. It is how many bytes + // of response headers are allow. Unlike the http2 spec, zero here + // means to use a default limit (currently 10MB). If you actually + // want to advertise an ulimited value to the peer, Transport + // interprets the highest possible value here (0xffffffff or 1<<32-1) + // to mean no limit. + MaxHeaderListSize uint32 + + // t1, if non-nil, is the standard library Transport using + // this transport. Its settings are used (but not its + // RoundTrip method, etc). + t1 *http.Transport + + connPoolOnce sync.Once + connPoolOrDef ClientConnPool // non-nil version of ConnPool } -type clientConn struct { - t *Transport - tconn *tls.Conn - tlsState *tls.ConnectionState - connKey []string // key(s) this connection is cached in, in t.conns +func (t *Transport) maxHeaderListSize() uint32 { + if t.MaxHeaderListSize == 0 { + return 10 << 20 + } + if t.MaxHeaderListSize == 0xffffffff { + return 0 + } + return t.MaxHeaderListSize +} +func (t *Transport) disableCompression() bool { + return t.DisableCompression || (t.t1 != nil && t.t1.DisableCompression) +} + +var errTransportVersion = errors.New("http2: ConfigureTransport is only supported starting at Go 1.6") + +// ConfigureTransport configures a net/http HTTP/1 Transport to use HTTP/2. +// It requires Go 1.6 or later and returns an error if the net/http package is too old +// or if t1 has already been HTTP/2-enabled. +func ConfigureTransport(t1 *http.Transport) error { + _, err := configureTransport(t1) // in configure_transport.go (go1.6) or not_go16.go + return err +} + +func (t *Transport) connPool() ClientConnPool { + t.connPoolOnce.Do(t.initConnPool) + return t.connPoolOrDef +} + +func (t *Transport) initConnPool() { + if t.ConnPool != nil { + t.connPoolOrDef = t.ConnPool + } else { + t.connPoolOrDef = &clientConnPool{t: t} + } +} + +// ClientConn is the state of a single HTTP/2 client connection to an +// HTTP/2 server. +type ClientConn struct { + t *Transport + tconn net.Conn // usually *tls.Conn, except specialized impls + tlsState *tls.ConnectionState // nil only for specialized impls + + // readLoop goroutine fields: readerDone chan struct{} // closed on error readerErr error // set before readerDone is closed - hdec *hpack.Decoder - nextRes *http.Response - mu sync.Mutex + mu sync.Mutex // guards following + cond *sync.Cond // hold mu; broadcast on flow/closed changes + flow flow // our conn-level flow control quota (cs.flow is per stream) + inflow flow // peer's conn-level flow control closed bool - goAway *GoAwayFrame // if non-nil, the GoAwayFrame we received - streams map[uint32]*clientStream + goAway *GoAwayFrame // if non-nil, the GoAwayFrame we received + streams map[uint32]*clientStream // client-initiated nextStreamID uint32 bw *bufio.Writer - werr error // first write error that has occurred br *bufio.Reader fr *Framer // Settings from peer: @@ -58,13 +159,78 @@ type clientConn struct { initialWindowSize uint32 hbuf bytes.Buffer // HPACK encoder writes into this henc *hpack.Encoder + freeBuf [][]byte + + wmu sync.Mutex // held while writing; acquire AFTER mu if holding both + werr error // first write error that has occurred } +// clientStream is the state for a single HTTP/2 stream. One of these +// is created for each Transport.RoundTrip call. type clientStream struct { - ID uint32 - resc chan resAndError - pw *io.PipeWriter - pr *io.PipeReader + cc *ClientConn + req *http.Request + ID uint32 + resc chan resAndError + bufPipe pipe // buffered pipe with the flow-controlled response payload + requestedGzip bool + + flow flow // guarded by cc.mu + inflow flow // guarded by cc.mu + bytesRemain int64 // -1 means unknown; owned by transportResponseBody.Read + readErr error // sticky read error; owned by transportResponseBody.Read + stopReqBody error // if non-nil, stop writing req body; guarded by cc.mu + + peerReset chan struct{} // closed on peer reset + resetErr error // populated before peerReset is closed + + done chan struct{} // closed when stream remove from cc.streams map; close calls guarded by cc.mu + + // owned by clientConnReadLoop: + pastHeaders bool // got HEADERS w/ END_HEADERS + pastTrailers bool // got second HEADERS frame w/ END_HEADERS + + trailer http.Header // accumulated trailers + resTrailer *http.Header // client's Response.Trailer +} + +// awaitRequestCancel runs in its own goroutine and waits for the user +// to either cancel a RoundTrip request (using the provided +// Request.Cancel channel), or for the request to be done (any way it +// might be removed from the cc.streams map: peer reset, successful +// completion, TCP connection breakage, etc) +func (cs *clientStream) awaitRequestCancel(cancel <-chan struct{}) { + if cancel == nil { + return + } + select { + case <-cancel: + cs.bufPipe.CloseWithError(errRequestCanceled) + cs.cc.writeStreamReset(cs.ID, ErrCodeCancel, nil) + case <-cs.done: + } +} + +// checkReset reports any error sent in a RST_STREAM frame by the +// server. +func (cs *clientStream) checkReset() error { + select { + case <-cs.peerReset: + return cs.resetErr + default: + return nil + } +} + +func (cs *clientStream) abortRequestBodyWrite(err error) { + if err == nil { + panic("nil error") + } + cc := cs.cc + cc.mu.Lock() + cs.stopReqBody = err + cc.cond.Broadcast() + cc.mu.Unlock() } type stickyErrWriter struct { @@ -81,30 +247,49 @@ func (sew stickyErrWriter) Write(p []byte) (n int, err error) { return } +var ErrNoCachedConn = errors.New("http2: no cached connection was available") + +// RoundTripOpt are options for the Transport.RoundTripOpt method. +type RoundTripOpt struct { + // OnlyCachedConn controls whether RoundTripOpt may + // create a new TCP connection. If set true and + // no cached connection is available, RoundTripOpt + // will return ErrNoCachedConn. + OnlyCachedConn bool +} + func (t *Transport) RoundTrip(req *http.Request) (*http.Response, error) { + return t.RoundTripOpt(req, RoundTripOpt{}) +} + +// authorityAddr returns a given authority (a host/IP, or host:port / ip:port) +// and returns a host:port. The port 443 is added if needed. +func authorityAddr(authority string) (addr string) { + if _, _, err := net.SplitHostPort(authority); err == nil { + return authority + } + return net.JoinHostPort(authority, "443") +} + +// RoundTripOpt is like RoundTrip, but takes options. +func (t *Transport) RoundTripOpt(req *http.Request, opt RoundTripOpt) (*http.Response, error) { if req.URL.Scheme != "https" { - if t.Fallback == nil { - return nil, errors.New("http2: unsupported scheme and no Fallback") - } - return t.Fallback.RoundTrip(req) - } - - host, port, err := net.SplitHostPort(req.URL.Host) - if err != nil { - host = req.URL.Host - port = "443" + return nil, errors.New("http2: unsupported scheme") } + addr := authorityAddr(req.URL.Host) for { - cc, err := t.getClientConn(host, port) + cc, err := t.connPool().GetClientConn(req, addr) if err != nil { + t.vlogf("http2: Transport failed to get client conn for %s: %v", addr, err) return nil, err } - res, err := cc.roundTrip(req) - if shouldRetryRequest(err) { // TODO: or clientconn is overloaded (too many outstanding requests)? + res, err := cc.RoundTrip(req) + if shouldRetryRequest(req, err) { continue } if err != nil { + t.vlogf("RoundTrip failure: %v", err) return nil, err } return res, nil @@ -115,106 +300,92 @@ func (t *Transport) RoundTrip(req *http.Request) (*http.Response, error) { // connected from previous requests but are now sitting idle. // It does not interrupt any connections currently in use. func (t *Transport) CloseIdleConnections() { - t.connMu.Lock() - defer t.connMu.Unlock() - for _, vv := range t.conns { - for _, cc := range vv { - cc.closeIfIdle() - } + if cp, ok := t.connPool().(*clientConnPool); ok { + cp.closeIdleConnections() } } -var errClientConnClosed = errors.New("http2: client conn is closed") +var ( + errClientConnClosed = errors.New("http2: client conn is closed") + errClientConnUnusable = errors.New("http2: client conn not usable") +) -func shouldRetryRequest(err error) bool { - // TODO: or GOAWAY graceful shutdown stuff - return err == errClientConnClosed +func shouldRetryRequest(req *http.Request, err error) bool { + // TODO: retry GET requests (no bodies) more aggressively, if shutdown + // before response. + return err == errClientConnUnusable } -func (t *Transport) removeClientConn(cc *clientConn) { - t.connMu.Lock() - defer t.connMu.Unlock() - for _, key := range cc.connKey { - vv, ok := t.conns[key] - if !ok { - continue - } - newList := filterOutClientConn(vv, cc) - if len(newList) > 0 { - t.conns[key] = newList - } else { - delete(t.conns, key) - } - } -} - -func filterOutClientConn(in []*clientConn, exclude *clientConn) []*clientConn { - out := in[:0] - for _, v := range in { - if v != exclude { - out = append(out, v) - } - } - return out -} - -func (t *Transport) getClientConn(host, port string) (*clientConn, error) { - t.connMu.Lock() - defer t.connMu.Unlock() - - key := net.JoinHostPort(host, port) - - for _, cc := range t.conns[key] { - if cc.canTakeNewRequest() { - return cc, nil - } - } - if t.conns == nil { - t.conns = make(map[string][]*clientConn) - } - cc, err := t.newClientConn(host, port, key) +func (t *Transport) dialClientConn(addr string) (*ClientConn, error) { + host, _, err := net.SplitHostPort(addr) if err != nil { return nil, err } - t.conns[key] = append(t.conns[key], cc) - return cc, nil -} - -func (t *Transport) newClientConn(host, port, key string) (*clientConn, error) { - cfg := &tls.Config{ - ServerName: host, - NextProtos: []string{NextProtoTLS}, - InsecureSkipVerify: t.InsecureTLSDial, - } - tconn, err := tls.Dial("tcp", net.JoinHostPort(host, port), cfg) + tconn, err := t.dialTLS()("tcp", addr, t.newTLSConfig(host)) if err != nil { return nil, err } - if err := tconn.Handshake(); err != nil { + return t.NewClientConn(tconn) +} + +func (t *Transport) newTLSConfig(host string) *tls.Config { + cfg := new(tls.Config) + if t.TLSClientConfig != nil { + *cfg = *t.TLSClientConfig + } + cfg.NextProtos = []string{NextProtoTLS} // TODO: don't override if already in list + cfg.ServerName = host + return cfg +} + +func (t *Transport) dialTLS() func(string, string, *tls.Config) (net.Conn, error) { + if t.DialTLS != nil { + return t.DialTLS + } + return t.dialTLSDefault +} + +func (t *Transport) dialTLSDefault(network, addr string, cfg *tls.Config) (net.Conn, error) { + cn, err := tls.Dial(network, addr, cfg) + if err != nil { return nil, err } - if !t.InsecureTLSDial { - if err := tconn.VerifyHostname(cfg.ServerName); err != nil { + if err := cn.Handshake(); err != nil { + return nil, err + } + if !cfg.InsecureSkipVerify { + if err := cn.VerifyHostname(cfg.ServerName); err != nil { return nil, err } } - state := tconn.ConnectionState() + state := cn.ConnectionState() if p := state.NegotiatedProtocol; p != NextProtoTLS { - // TODO(bradfitz): fall back to Fallback - return nil, fmt.Errorf("bad protocol: %v", p) + return nil, fmt.Errorf("http2: unexpected ALPN protocol %q; want %q", p, NextProtoTLS) } if !state.NegotiatedProtocolIsMutual { - return nil, errors.New("could not negotiate protocol mutually") + return nil, errors.New("http2: could not negotiate protocol mutually") } - if _, err := tconn.Write(clientPreface); err != nil { + return cn, nil +} + +// disableKeepAlives reports whether connections should be closed as +// soon as possible after handling the first request. +func (t *Transport) disableKeepAlives() bool { + return t.t1 != nil && t.t1.DisableKeepAlives +} + +func (t *Transport) NewClientConn(c net.Conn) (*ClientConn, error) { + if VerboseLogs { + t.vlogf("http2: Transport creating client conn to %v", c.RemoteAddr()) + } + if _, err := c.Write(clientPreface); err != nil { + t.vlogf("client preface write error: %v", err) return nil, err } - cc := &clientConn{ + cc := &ClientConn{ t: t, - tconn: tconn, - connKey: []string{key}, // TODO: cert's validated hostnames too - tlsState: &state, + tconn: c, readerDone: make(chan struct{}), nextStreamID: 1, maxFrameSize: 16 << 10, // spec default @@ -222,14 +393,34 @@ func (t *Transport) newClientConn(host, port, key string) (*clientConn, error) { maxConcurrentStreams: 1000, // "infinite", per spec. 1000 seems good enough. streams: make(map[uint32]*clientStream), } - cc.bw = bufio.NewWriter(stickyErrWriter{tconn, &cc.werr}) - cc.br = bufio.NewReader(tconn) + cc.cond = sync.NewCond(&cc.mu) + cc.flow.add(int32(initialWindowSize)) + + // TODO: adjust this writer size to account for frame size + + // MTU + crypto/tls record padding. + cc.bw = bufio.NewWriter(stickyErrWriter{c, &cc.werr}) + cc.br = bufio.NewReader(c) cc.fr = NewFramer(cc.bw, cc.br) + + // TODO: SetMaxDynamicTableSize, SetMaxDynamicTableSizeLimit on + // henc in response to SETTINGS frames? cc.henc = hpack.NewEncoder(&cc.hbuf) - cc.fr.WriteSettings() - // TODO: re-send more conn-level flow control tokens when server uses all these. - cc.fr.WriteWindowUpdate(0, 1<<30) // um, 0x7fffffff doesn't work to Google? it hangs? + if cs, ok := c.(connectionStater); ok { + state := cs.ConnectionState() + cc.tlsState = &state + } + + initialSettings := []Setting{ + Setting{ID: SettingEnablePush, Val: 0}, + Setting{ID: SettingInitialWindowSize, Val: transportDefaultStreamFlow}, + } + if max := t.maxHeaderListSize(); max != 0 { + initialSettings = append(initialSettings, Setting{ID: SettingMaxHeaderListSize, Val: max}) + } + cc.fr.WriteSettings(initialSettings...) + cc.fr.WriteWindowUpdate(0, transportDefaultConnFlow) + cc.inflow.add(transportDefaultConnFlow + initialWindowSize) cc.bw.Flush() if cc.werr != nil { return nil, cc.werr @@ -256,33 +447,35 @@ func (t *Transport) newClientConn(host, port, key string) (*clientConn, error) { case SettingInitialWindowSize: cc.initialWindowSize = s.Val default: - // TODO(bradfitz): handle more - log.Printf("Unhandled Setting: %v", s) + // TODO(bradfitz): handle more; at least SETTINGS_HEADER_TABLE_SIZE? + t.vlogf("Unhandled Setting: %v", s) } return nil }) - // TODO: figure out henc size - cc.hdec = hpack.NewDecoder(initialHeaderTableSize, cc.onNewHeaderField) go cc.readLoop() return cc, nil } -func (cc *clientConn) setGoAway(f *GoAwayFrame) { +func (cc *ClientConn) setGoAway(f *GoAwayFrame) { cc.mu.Lock() defer cc.mu.Unlock() cc.goAway = f } -func (cc *clientConn) canTakeNewRequest() bool { +func (cc *ClientConn) CanTakeNewRequest() bool { cc.mu.Lock() defer cc.mu.Unlock() - return cc.goAway == nil && + return cc.canTakeNewRequestLocked() +} + +func (cc *ClientConn) canTakeNewRequestLocked() bool { + return cc.goAway == nil && !cc.closed && int64(len(cc.streams)+1) < int64(cc.maxConcurrentStreams) && cc.nextStreamID < 2147483647 } -func (cc *clientConn) closeIfIdle() { +func (cc *ClientConn) closeIfIdle() { cc.mu.Lock() if len(cc.streams) > 0 { cc.mu.Unlock() @@ -295,96 +488,536 @@ func (cc *clientConn) closeIfIdle() { cc.tconn.Close() } -func (cc *clientConn) roundTrip(req *http.Request) (*http.Response, error) { - cc.mu.Lock() +const maxAllocFrameSize = 512 << 10 - if cc.closed { +// frameBuffer returns a scratch buffer suitable for writing DATA frames. +// They're capped at the min of the peer's max frame size or 512KB +// (kinda arbitrarily), but definitely capped so we don't allocate 4GB +// bufers. +func (cc *ClientConn) frameScratchBuffer() []byte { + cc.mu.Lock() + size := cc.maxFrameSize + if size > maxAllocFrameSize { + size = maxAllocFrameSize + } + for i, buf := range cc.freeBuf { + if len(buf) >= int(size) { + cc.freeBuf[i] = nil + cc.mu.Unlock() + return buf[:size] + } + } + cc.mu.Unlock() + return make([]byte, size) +} + +func (cc *ClientConn) putFrameScratchBuffer(buf []byte) { + cc.mu.Lock() + defer cc.mu.Unlock() + const maxBufs = 4 // arbitrary; 4 concurrent requests per conn? investigate. + if len(cc.freeBuf) < maxBufs { + cc.freeBuf = append(cc.freeBuf, buf) + return + } + for i, old := range cc.freeBuf { + if old == nil { + cc.freeBuf[i] = buf + return + } + } + // forget about it. +} + +// errRequestCanceled is a copy of net/http's errRequestCanceled because it's not +// exported. At least they'll be DeepEqual for h1-vs-h2 comparisons tests. +var errRequestCanceled = errors.New("net/http: request canceled") + +func commaSeparatedTrailers(req *http.Request) (string, error) { + keys := make([]string, 0, len(req.Trailer)) + for k := range req.Trailer { + k = http.CanonicalHeaderKey(k) + switch k { + case "Transfer-Encoding", "Trailer", "Content-Length": + return "", &badStringError{"invalid Trailer key", k} + } + keys = append(keys, k) + } + if len(keys) > 0 { + sort.Strings(keys) + // TODO: could do better allocation-wise here, but trailers are rare, + // so being lazy for now. + return strings.Join(keys, ","), nil + } + return "", nil +} + +func (cc *ClientConn) responseHeaderTimeout() time.Duration { + if cc.t.t1 != nil { + return cc.t.t1.ResponseHeaderTimeout + } + // No way to do this (yet?) with just an http2.Transport. Probably + // no need. Request.Cancel this is the new way. We only need to support + // this for compatibility with the old http.Transport fields when + // we're doing transparent http2. + return 0 +} + +// checkConnHeaders checks whether req has any invalid connection-level headers. +// per RFC 7540 section 8.1.2.2: Connection-Specific Header Fields. +// Certain headers are special-cased as okay but not transmitted later. +func checkConnHeaders(req *http.Request) error { + if v := req.Header.Get("Upgrade"); v != "" { + return errors.New("http2: invalid Upgrade request header") + } + if v := req.Header.Get("Transfer-Encoding"); (v != "" && v != "chunked") || len(req.Header["Transfer-Encoding"]) > 1 { + return errors.New("http2: invalid Transfer-Encoding request header") + } + if v := req.Header.Get("Connection"); (v != "" && v != "close" && v != "keep-alive") || len(req.Header["Connection"]) > 1 { + return errors.New("http2: invalid Connection request header") + } + return nil +} + +func (cc *ClientConn) RoundTrip(req *http.Request) (*http.Response, error) { + if err := checkConnHeaders(req); err != nil { + return nil, err + } + + trailers, err := commaSeparatedTrailers(req) + if err != nil { + return nil, err + } + hasTrailers := trailers != "" + + var body io.Reader = req.Body + contentLen := req.ContentLength + if req.Body != nil && contentLen == 0 { + // Test to see if it's actually zero or just unset. + var buf [1]byte + n, rerr := io.ReadFull(body, buf[:]) + if rerr != nil && rerr != io.EOF { + contentLen = -1 + body = errorReader{rerr} + } else if n == 1 { + // Oh, guess there is data in this Body Reader after all. + // The ContentLength field just wasn't set. + // Stich the Body back together again, re-attaching our + // consumed byte. + contentLen = -1 + body = io.MultiReader(bytes.NewReader(buf[:]), body) + } else { + // Body is actually empty. + body = nil + } + } + + cc.mu.Lock() + if cc.closed || !cc.canTakeNewRequestLocked() { cc.mu.Unlock() - return nil, errClientConnClosed + return nil, errClientConnUnusable } cs := cc.newStream() - hasBody := false // TODO + cs.req = req + hasBody := body != nil - // we send: HEADERS[+CONTINUATION] + (DATA?) - hdrs := cc.encodeHeaders(req) - first := true - for len(hdrs) > 0 { + // TODO(bradfitz): this is a copy of the logic in net/http. Unify somewhere? + if !cc.t.disableCompression() && + req.Header.Get("Accept-Encoding") == "" && + req.Header.Get("Range") == "" && + req.Method != "HEAD" { + // Request gzip only, not deflate. Deflate is ambiguous and + // not as universally supported anyway. + // See: http://www.gzip.org/zlib/zlib_faq.html#faq38 + // + // Note that we don't request this for HEAD requests, + // due to a bug in nginx: + // http://trac.nginx.org/nginx/ticket/358 + // https://golang.org/issue/5522 + // + // We don't request gzip if the request is for a range, since + // auto-decoding a portion of a gzipped document will just fail + // anyway. See https://golang.org/issue/8923 + cs.requestedGzip = true + } + + // we send: HEADERS{1}, CONTINUATION{0,} + DATA{0,} (DATA is + // sent by writeRequestBody below, along with any Trailers, + // again in form HEADERS{1}, CONTINUATION{0,}) + hdrs := cc.encodeHeaders(req, cs.requestedGzip, trailers, contentLen) + cc.wmu.Lock() + endStream := !hasBody && !hasTrailers + werr := cc.writeHeaders(cs.ID, endStream, hdrs) + cc.wmu.Unlock() + cc.mu.Unlock() + + if werr != nil { + if hasBody { + req.Body.Close() // per RoundTripper contract + } + cc.forgetStreamID(cs.ID) + // Don't bother sending a RST_STREAM (our write already failed; + // no need to keep writing) + return nil, werr + } + + var respHeaderTimer <-chan time.Time + var bodyCopyErrc chan error // result of body copy + if hasBody { + bodyCopyErrc = make(chan error, 1) + go func() { + bodyCopyErrc <- cs.writeRequestBody(body, req.Body) + }() + } else { + if d := cc.responseHeaderTimeout(); d != 0 { + timer := time.NewTimer(d) + defer timer.Stop() + respHeaderTimer = timer.C + } + } + + readLoopResCh := cs.resc + requestCanceledCh := requestCancel(req) + bodyWritten := false + + for { + select { + case re := <-readLoopResCh: + res := re.res + if re.err != nil || res.StatusCode > 299 { + // On error or status code 3xx, 4xx, 5xx, etc abort any + // ongoing write, assuming that the server doesn't care + // about our request body. If the server replied with 1xx or + // 2xx, however, then assume the server DOES potentially + // want our body (e.g. full-duplex streaming: + // golang.org/issue/13444). If it turns out the server + // doesn't, they'll RST_STREAM us soon enough. This is a + // heuristic to avoid adding knobs to Transport. Hopefully + // we can keep it. + cs.abortRequestBodyWrite(errStopReqBodyWrite) + } + if re.err != nil { + cc.forgetStreamID(cs.ID) + return nil, re.err + } + res.Request = req + res.TLS = cc.tlsState + return res, nil + case <-respHeaderTimer: + cc.forgetStreamID(cs.ID) + if !hasBody || bodyWritten { + cc.writeStreamReset(cs.ID, ErrCodeCancel, nil) + } else { + cs.abortRequestBodyWrite(errStopReqBodyWriteAndCancel) + } + return nil, errTimeout + case <-requestCanceledCh: + cc.forgetStreamID(cs.ID) + if !hasBody || bodyWritten { + cc.writeStreamReset(cs.ID, ErrCodeCancel, nil) + } else { + cs.abortRequestBodyWrite(errStopReqBodyWriteAndCancel) + } + return nil, errRequestCanceled + case <-cs.peerReset: + // processResetStream already removed the + // stream from the streams map; no need for + // forgetStreamID. + return nil, cs.resetErr + case err := <-bodyCopyErrc: + if err != nil { + return nil, err + } + bodyWritten = true + if d := cc.responseHeaderTimeout(); d != 0 { + timer := time.NewTimer(d) + defer timer.Stop() + respHeaderTimer = timer.C + } + } + } +} + +// requires cc.wmu be held +func (cc *ClientConn) writeHeaders(streamID uint32, endStream bool, hdrs []byte) error { + first := true // first frame written (HEADERS is first, then CONTINUATION) + frameSize := int(cc.maxFrameSize) + for len(hdrs) > 0 && cc.werr == nil { chunk := hdrs - if len(chunk) > int(cc.maxFrameSize) { - chunk = chunk[:cc.maxFrameSize] + if len(chunk) > frameSize { + chunk = chunk[:frameSize] } hdrs = hdrs[len(chunk):] endHeaders := len(hdrs) == 0 if first { cc.fr.WriteHeaders(HeadersFrameParam{ - StreamID: cs.ID, + StreamID: streamID, BlockFragment: chunk, - EndStream: !hasBody, + EndStream: endStream, EndHeaders: endHeaders, }) first = false } else { - cc.fr.WriteContinuation(cs.ID, endHeaders, chunk) + cc.fr.WriteContinuation(streamID, endHeaders, chunk) } } + // TODO(bradfitz): this Flush could potentially block (as + // could the WriteHeaders call(s) above), which means they + // wouldn't respond to Request.Cancel being readable. That's + // rare, but this should probably be in a goroutine. cc.bw.Flush() - werr := cc.werr - cc.mu.Unlock() - - if hasBody { - // TODO: write data. and it should probably be interleaved: - // go ... io.Copy(dataFrameWriter{cc, cs, ...}, req.Body) ... etc - } - - if werr != nil { - return nil, werr - } - - re := <-cs.resc - if re.err != nil { - return nil, re.err - } - res := re.res - res.Request = req - res.TLS = cc.tlsState - return res, nil + return cc.werr } +// internal error values; they don't escape to callers +var ( + // abort request body write; don't send cancel + errStopReqBodyWrite = errors.New("http2: aborting request body write") + + // abort request body write, but send stream reset of cancel. + errStopReqBodyWriteAndCancel = errors.New("http2: canceling request") +) + +func (cs *clientStream) writeRequestBody(body io.Reader, bodyCloser io.Closer) (err error) { + cc := cs.cc + sentEnd := false // whether we sent the final DATA frame w/ END_STREAM + buf := cc.frameScratchBuffer() + defer cc.putFrameScratchBuffer(buf) + + defer func() { + // TODO: write h12Compare test showing whether + // Request.Body is closed by the Transport, + // and in multiple cases: server replies <=299 and >299 + // while still writing request body + cerr := bodyCloser.Close() + if err == nil { + err = cerr + } + }() + + req := cs.req + hasTrailers := req.Trailer != nil + + var sawEOF bool + for !sawEOF { + n, err := body.Read(buf) + if err == io.EOF { + sawEOF = true + err = nil + } else if err != nil { + return err + } + + remain := buf[:n] + for len(remain) > 0 && err == nil { + var allowed int32 + allowed, err = cs.awaitFlowControl(len(remain)) + switch { + case err == errStopReqBodyWrite: + return err + case err == errStopReqBodyWriteAndCancel: + cc.writeStreamReset(cs.ID, ErrCodeCancel, nil) + return err + case err != nil: + return err + } + cc.wmu.Lock() + data := remain[:allowed] + remain = remain[allowed:] + sentEnd = sawEOF && len(remain) == 0 && !hasTrailers + err = cc.fr.WriteData(cs.ID, sentEnd, data) + if err == nil { + // TODO(bradfitz): this flush is for latency, not bandwidth. + // Most requests won't need this. Make this opt-in or opt-out? + // Use some heuristic on the body type? Nagel-like timers? + // Based on 'n'? Only last chunk of this for loop, unless flow control + // tokens are low? For now, always: + err = cc.bw.Flush() + } + cc.wmu.Unlock() + } + if err != nil { + return err + } + } + + cc.wmu.Lock() + if !sentEnd { + var trls []byte + if hasTrailers { + cc.mu.Lock() + trls = cc.encodeTrailers(req) + cc.mu.Unlock() + } + + // Avoid forgetting to send an END_STREAM if the encoded + // trailers are 0 bytes. Both results produce and END_STREAM. + if len(trls) > 0 { + err = cc.writeHeaders(cs.ID, true, trls) + } else { + err = cc.fr.WriteData(cs.ID, true, nil) + } + } + if ferr := cc.bw.Flush(); ferr != nil && err == nil { + err = ferr + } + cc.wmu.Unlock() + + return err +} + +// awaitFlowControl waits for [1, min(maxBytes, cc.cs.maxFrameSize)] flow +// control tokens from the server. +// It returns either the non-zero number of tokens taken or an error +// if the stream is dead. +func (cs *clientStream) awaitFlowControl(maxBytes int) (taken int32, err error) { + cc := cs.cc + cc.mu.Lock() + defer cc.mu.Unlock() + for { + if cc.closed { + return 0, errClientConnClosed + } + if cs.stopReqBody != nil { + return 0, cs.stopReqBody + } + if err := cs.checkReset(); err != nil { + return 0, err + } + if a := cs.flow.available(); a > 0 { + take := a + if int(take) > maxBytes { + + take = int32(maxBytes) // can't truncate int; take is int32 + } + if take > int32(cc.maxFrameSize) { + take = int32(cc.maxFrameSize) + } + cs.flow.take(take) + return take, nil + } + cc.cond.Wait() + } +} + +type badStringError struct { + what string + str string +} + +func (e *badStringError) Error() string { return fmt.Sprintf("%s %q", e.what, e.str) } + // requires cc.mu be held. -func (cc *clientConn) encodeHeaders(req *http.Request) []byte { +func (cc *ClientConn) encodeHeaders(req *http.Request, addGzipHeader bool, trailers string, contentLength int64) []byte { cc.hbuf.Reset() - // TODO(bradfitz): figure out :authority-vs-Host stuff between http2 and Go host := req.Host if host == "" { host = req.URL.Host } - path := req.URL.Path - if path == "" { - path = "/" + // 8.1.2.3 Request Pseudo-Header Fields + // The :path pseudo-header field includes the path and query parts of the + // target URI (the path-absolute production and optionally a '?' character + // followed by the query production (see Sections 3.3 and 3.4 of + // [RFC3986]). + cc.writeHeader(":authority", host) + cc.writeHeader(":method", req.Method) + if req.Method != "CONNECT" { + cc.writeHeader(":path", req.URL.RequestURI()) + cc.writeHeader(":scheme", "https") + } + if trailers != "" { + cc.writeHeader("trailer", trailers) } - cc.writeHeader(":authority", host) // probably not right for all sites - cc.writeHeader(":method", req.Method) - cc.writeHeader(":path", path) - cc.writeHeader(":scheme", "https") - + var didUA bool for k, vv := range req.Header { lowKey := strings.ToLower(k) - if lowKey == "host" { + switch lowKey { + case "host", "content-length": + // Host is :authority, already sent. + // Content-Length is automatic, set below. continue + case "connection", "proxy-connection", "transfer-encoding", "upgrade": + // Per 8.1.2.2 Connection-Specific Header + // Fields, don't send connection-specific + // fields. We deal with these earlier in + // RoundTrip, deciding whether they're + // error-worthy, but we don't want to mutate + // the user's *Request so at this point, just + // skip over them at this point. + continue + case "user-agent": + // Match Go's http1 behavior: at most one + // User-Agent. If set to nil or empty string, + // then omit it. Otherwise if not mentioned, + // include the default (below). + didUA = true + if len(vv) < 1 { + continue + } + vv = vv[:1] + if vv[0] == "" { + continue + } } for _, v := range vv { cc.writeHeader(lowKey, v) } } + if shouldSendReqContentLength(req.Method, contentLength) { + cc.writeHeader("content-length", strconv.FormatInt(contentLength, 10)) + } + if addGzipHeader { + cc.writeHeader("accept-encoding", "gzip") + } + if !didUA { + cc.writeHeader("user-agent", defaultUserAgent) + } + return cc.hbuf.Bytes() +} + +// shouldSendReqContentLength reports whether the http2.Transport should send +// a "content-length" request header. This logic is basically a copy of the net/http +// transferWriter.shouldSendContentLength. +// The contentLength is the corrected contentLength (so 0 means actually 0, not unknown). +// -1 means unknown. +func shouldSendReqContentLength(method string, contentLength int64) bool { + if contentLength > 0 { + return true + } + if contentLength < 0 { + return false + } + // For zero bodies, whether we send a content-length depends on the method. + // It also kinda doesn't matter for http2 either way, with END_STREAM. + switch method { + case "POST", "PUT", "PATCH": + return true + default: + return false + } +} + +// requires cc.mu be held. +func (cc *ClientConn) encodeTrailers(req *http.Request) []byte { + cc.hbuf.Reset() + for k, vv := range req.Trailer { + // Transfer-Encoding, etc.. have already been filter at the + // start of RoundTrip + lowKey := strings.ToLower(k) + for _, v := range vv { + cc.writeHeader(lowKey, v) + } + } return cc.hbuf.Bytes() } -func (cc *clientConn) writeHeader(name, value string) { - log.Printf("sending %q = %q", name, value) +func (cc *ClientConn) writeHeader(name, value string) { + if VerboseLogs { + log.Printf("http2: Transport encoding header %q = %q", name, value) + } cc.henc.WriteField(hpack.HeaderField{Name: name, Value: value}) } @@ -394,160 +1027,724 @@ type resAndError struct { } // requires cc.mu be held. -func (cc *clientConn) newStream() *clientStream { +func (cc *ClientConn) newStream() *clientStream { cs := &clientStream{ - ID: cc.nextStreamID, - resc: make(chan resAndError, 1), + cc: cc, + ID: cc.nextStreamID, + resc: make(chan resAndError, 1), + peerReset: make(chan struct{}), + done: make(chan struct{}), } + cs.flow.add(int32(cc.initialWindowSize)) + cs.flow.setConnFlow(&cc.flow) + cs.inflow.add(transportDefaultStreamFlow) + cs.inflow.setConnFlow(&cc.inflow) cc.nextStreamID += 2 cc.streams[cs.ID] = cs return cs } -func (cc *clientConn) streamByID(id uint32, andRemove bool) *clientStream { +func (cc *ClientConn) forgetStreamID(id uint32) { + cc.streamByID(id, true) +} + +func (cc *ClientConn) streamByID(id uint32, andRemove bool) *clientStream { cc.mu.Lock() defer cc.mu.Unlock() cs := cc.streams[id] - if andRemove { + if andRemove && cs != nil && !cc.closed { delete(cc.streams, id) + close(cs.done) } return cs } -// runs in its own goroutine. -func (cc *clientConn) readLoop() { - defer cc.t.removeClientConn(cc) +// clientConnReadLoop is the state owned by the clientConn's frame-reading readLoop. +type clientConnReadLoop struct { + cc *ClientConn + activeRes map[uint32]*clientStream // keyed by streamID + closeWhenIdle bool + + hdec *hpack.Decoder + + // Fields reset on each HEADERS: + nextRes *http.Response + sawRegHeader bool // saw non-pseudo header + reqMalformed error // non-nil once known to be malformed + lastHeaderEndsStream bool + headerListSize int64 // actually uint32, but easier math this way +} + +// readLoop runs in its own goroutine and reads and dispatches frames. +func (cc *ClientConn) readLoop() { + rl := &clientConnReadLoop{ + cc: cc, + activeRes: make(map[uint32]*clientStream), + } + rl.hdec = hpack.NewDecoder(initialHeaderTableSize, rl.onNewHeaderField) + + defer rl.cleanup() + cc.readerErr = rl.run() + if ce, ok := cc.readerErr.(ConnectionError); ok { + cc.wmu.Lock() + cc.fr.WriteGoAway(0, ErrCode(ce), nil) + cc.wmu.Unlock() + } +} + +func (rl *clientConnReadLoop) cleanup() { + cc := rl.cc + defer cc.tconn.Close() + defer cc.t.connPool().MarkDead(cc) defer close(cc.readerDone) - activeRes := map[uint32]*clientStream{} // keyed by streamID // Close any response bodies if the server closes prematurely. // TODO: also do this if we've written the headers but not // gotten a response yet. - defer func() { - err := cc.readerErr - if err == io.EOF { - err = io.ErrUnexpectedEOF + err := cc.readerErr + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + cc.mu.Lock() + for _, cs := range rl.activeRes { + cs.bufPipe.CloseWithError(err) + } + for _, cs := range cc.streams { + select { + case cs.resc <- resAndError{err: err}: + default: } - for _, cs := range activeRes { - cs.pw.CloseWithError(err) - } - }() - - // continueStreamID is the stream ID we're waiting for - // continuation frames for. - var continueStreamID uint32 + close(cs.done) + } + cc.closed = true + cc.cond.Broadcast() + cc.mu.Unlock() +} +func (rl *clientConnReadLoop) run() error { + cc := rl.cc + rl.closeWhenIdle = cc.t.disableKeepAlives() + gotReply := false // ever saw a reply for { f, err := cc.fr.ReadFrame() if err != nil { - cc.readerErr = err - return + cc.vlogf("Transport readFrame error: (%T) %v", err, err) } - log.Printf("Transport received %v: %#v", f.Header(), f) - - streamID := f.Header().StreamID - - _, isContinue := f.(*ContinuationFrame) - if isContinue { - if streamID != continueStreamID { - log.Printf("Protocol violation: got CONTINUATION with id %d; want %d", streamID, continueStreamID) - cc.readerErr = ConnectionError(ErrCodeProtocol) - return - } - } else if continueStreamID != 0 { - // Continue frames need to be adjacent in the stream - // and we were in the middle of headers. - log.Printf("Protocol violation: got %T for stream %d, want CONTINUATION for %d", f, streamID, continueStreamID) - cc.readerErr = ConnectionError(ErrCodeProtocol) - return + if se, ok := err.(StreamError); ok { + // TODO: deal with stream errors from the framer. + return se + } else if err != nil { + return err } - - if streamID%2 == 0 { - // Ignore streams pushed from the server for now. - // These always have an even stream id. - continue - } - streamEnded := false - if ff, ok := f.(streamEnder); ok { - streamEnded = ff.StreamEnded() - } - - cs := cc.streamByID(streamID, streamEnded) - if cs == nil { - log.Printf("Received frame for untracked stream ID %d", streamID) - continue + if VerboseLogs { + cc.vlogf("http2: Transport received %s", summarizeFrame(f)) } + maybeIdle := false // whether frame might transition us to idle switch f := f.(type) { case *HeadersFrame: - cc.nextRes = &http.Response{ - Proto: "HTTP/2.0", - ProtoMajor: 2, - Header: make(http.Header), - } - cs.pr, cs.pw = io.Pipe() - cc.hdec.Write(f.HeaderBlockFragment()) + err = rl.processHeaders(f) + maybeIdle = true + gotReply = true case *ContinuationFrame: - cc.hdec.Write(f.HeaderBlockFragment()) + err = rl.processContinuation(f) + maybeIdle = true case *DataFrame: - log.Printf("DATA: %q", f.Data()) - cs.pw.Write(f.Data()) + err = rl.processData(f) + maybeIdle = true case *GoAwayFrame: - cc.t.removeClientConn(cc) - if f.ErrCode != 0 { - // TODO: deal with GOAWAY more. particularly the error code - log.Printf("transport got GOAWAY with error code = %v", f.ErrCode) - } - cc.setGoAway(f) + err = rl.processGoAway(f) + maybeIdle = true + case *RSTStreamFrame: + err = rl.processResetStream(f) + maybeIdle = true + case *SettingsFrame: + err = rl.processSettings(f) + case *PushPromiseFrame: + err = rl.processPushPromise(f) + case *WindowUpdateFrame: + err = rl.processWindowUpdate(f) + case *PingFrame: + err = rl.processPing(f) default: - log.Printf("Transport: unhandled response frame type %T", f) + cc.logf("Transport: unhandled response frame type %T", f) } - headersEnded := false - if he, ok := f.(headersEnder); ok { - headersEnded = he.HeadersEnded() - if headersEnded { - continueStreamID = 0 - } else { - continueStreamID = streamID - } + if err != nil { + return err } - - if streamEnded { - cs.pw.Close() - delete(activeRes, streamID) - } - if headersEnded { - if cs == nil { - panic("couldn't find stream") // TODO be graceful - } - // TODO: set the Body to one which notes the - // Close and also sends the server a - // RST_STREAM - cc.nextRes.Body = cs.pr - res := cc.nextRes - activeRes[streamID] = cs - cs.resc <- resAndError{res: res} + if rl.closeWhenIdle && gotReply && maybeIdle && len(rl.activeRes) == 0 { + cc.closeIfIdle() } } } -func (cc *clientConn) onNewHeaderField(f hpack.HeaderField) { - // TODO: verifiy pseudo headers come before non-pseudo headers - // TODO: verifiy the status is set - log.Printf("Header field: %+v", f) - if f.Name == ":status" { - code, err := strconv.Atoi(f.Value) - if err != nil { - panic("TODO: be graceful") +func (rl *clientConnReadLoop) processHeaders(f *HeadersFrame) error { + rl.sawRegHeader = false + rl.reqMalformed = nil + rl.lastHeaderEndsStream = f.StreamEnded() + rl.headerListSize = 0 + rl.nextRes = &http.Response{ + Proto: "HTTP/2.0", + ProtoMajor: 2, + Header: make(http.Header), + } + rl.hdec.SetEmitEnabled(true) + return rl.processHeaderBlockFragment(f.HeaderBlockFragment(), f.StreamID, f.HeadersEnded()) +} + +func (rl *clientConnReadLoop) processContinuation(f *ContinuationFrame) error { + return rl.processHeaderBlockFragment(f.HeaderBlockFragment(), f.StreamID, f.HeadersEnded()) +} + +func (rl *clientConnReadLoop) processHeaderBlockFragment(frag []byte, streamID uint32, finalFrag bool) error { + cc := rl.cc + streamEnded := rl.lastHeaderEndsStream + cs := cc.streamByID(streamID, streamEnded && finalFrag) + if cs == nil { + // We'd get here if we canceled a request while the + // server was mid-way through replying with its + // headers. (The case of a CONTINUATION arriving + // without HEADERS would be rejected earlier by the + // Framer). So if this was just something we canceled, + // ignore it. + return nil + } + if cs.pastHeaders { + rl.hdec.SetEmitFunc(func(f hpack.HeaderField) { rl.onNewTrailerField(cs, f) }) + } else { + rl.hdec.SetEmitFunc(rl.onNewHeaderField) + } + _, err := rl.hdec.Write(frag) + if err != nil { + return ConnectionError(ErrCodeCompression) + } + if finalFrag { + if err := rl.hdec.Close(); err != nil { + return ConnectionError(ErrCodeCompression) } - cc.nextRes.Status = f.Value + " " + http.StatusText(code) - cc.nextRes.StatusCode = code + } + + if !finalFrag { + return nil + } + + if !cs.pastHeaders { + cs.pastHeaders = true + } else { + // We're dealing with trailers. (and specifically the + // final frame of headers) + if cs.pastTrailers { + // Too many HEADERS frames for this stream. + return ConnectionError(ErrCodeProtocol) + } + cs.pastTrailers = true + if !streamEnded { + // We expect that any header block fragment + // frame for trailers with END_HEADERS also + // has END_STREAM. + return ConnectionError(ErrCodeProtocol) + } + rl.endStream(cs) + return nil + } + + if rl.reqMalformed != nil { + cs.resc <- resAndError{err: rl.reqMalformed} + rl.cc.writeStreamReset(cs.ID, ErrCodeProtocol, rl.reqMalformed) + return nil + } + + res := rl.nextRes + + if res.StatusCode == 100 { + // Just skip 100-continue response headers for now. + // TODO: golang.org/issue/13851 for doing it properly. + cs.pastHeaders = false // do it all again + return nil + } + + if !streamEnded || cs.req.Method == "HEAD" { + res.ContentLength = -1 + if clens := res.Header["Content-Length"]; len(clens) == 1 { + if clen64, err := strconv.ParseInt(clens[0], 10, 64); err == nil { + res.ContentLength = clen64 + } else { + // TODO: care? unlike http/1, it won't mess up our framing, so it's + // more safe smuggling-wise to ignore. + } + } else if len(clens) > 1 { + // TODO: care? unlike http/1, it won't mess up our framing, so it's + // more safe smuggling-wise to ignore. + } + } + + if streamEnded { + res.Body = noBody + } else { + buf := new(bytes.Buffer) // TODO(bradfitz): recycle this garbage + cs.bufPipe = pipe{b: buf} + cs.bytesRemain = res.ContentLength + res.Body = transportResponseBody{cs} + go cs.awaitRequestCancel(requestCancel(cs.req)) + + if cs.requestedGzip && res.Header.Get("Content-Encoding") == "gzip" { + res.Header.Del("Content-Encoding") + res.Header.Del("Content-Length") + res.ContentLength = -1 + res.Body = &gzipReader{body: res.Body} + } + rl.activeRes[cs.ID] = cs + } + + cs.resTrailer = &res.Trailer + cs.resc <- resAndError{res: res} + rl.nextRes = nil // unused now; will be reset next HEADERS frame + return nil +} + +// transportResponseBody is the concrete type of Transport.RoundTrip's +// Response.Body. It is an io.ReadCloser. On Read, it reads from cs.body. +// On Close it sends RST_STREAM if EOF wasn't already seen. +type transportResponseBody struct { + cs *clientStream +} + +func (b transportResponseBody) Read(p []byte) (n int, err error) { + cs := b.cs + cc := cs.cc + + if cs.readErr != nil { + return 0, cs.readErr + } + n, err = b.cs.bufPipe.Read(p) + if cs.bytesRemain != -1 { + if int64(n) > cs.bytesRemain { + n = int(cs.bytesRemain) + if err == nil { + err = errors.New("net/http: server replied with more than declared Content-Length; truncated") + cc.writeStreamReset(cs.ID, ErrCodeProtocol, err) + } + cs.readErr = err + return int(cs.bytesRemain), err + } + cs.bytesRemain -= int64(n) + if err == io.EOF && cs.bytesRemain > 0 { + err = io.ErrUnexpectedEOF + cs.readErr = err + return n, err + } + } + if n == 0 { + // No flow control tokens to send back. + return + } + + cc.mu.Lock() + defer cc.mu.Unlock() + + var connAdd, streamAdd int32 + // Check the conn-level first, before the stream-level. + if v := cc.inflow.available(); v < transportDefaultConnFlow/2 { + connAdd = transportDefaultConnFlow - v + cc.inflow.add(connAdd) + } + if err == nil { // No need to refresh if the stream is over or failed. + if v := cs.inflow.available(); v < transportDefaultStreamFlow-transportDefaultStreamMinRefresh { + streamAdd = transportDefaultStreamFlow - v + cs.inflow.add(streamAdd) + } + } + if connAdd != 0 || streamAdd != 0 { + cc.wmu.Lock() + defer cc.wmu.Unlock() + if connAdd != 0 { + cc.fr.WriteWindowUpdate(0, mustUint31(connAdd)) + } + if streamAdd != 0 { + cc.fr.WriteWindowUpdate(cs.ID, mustUint31(streamAdd)) + } + cc.bw.Flush() + } + return +} + +var errClosedResponseBody = errors.New("http2: response body closed") + +func (b transportResponseBody) Close() error { + cs := b.cs + if cs.bufPipe.Err() != io.EOF { + // TODO: write test for this + cs.cc.writeStreamReset(cs.ID, ErrCodeCancel, nil) + } + cs.bufPipe.BreakWithError(errClosedResponseBody) + return nil +} + +func (rl *clientConnReadLoop) processData(f *DataFrame) error { + cc := rl.cc + cs := cc.streamByID(f.StreamID, f.StreamEnded()) + if cs == nil { + cc.mu.Lock() + neverSent := cc.nextStreamID + cc.mu.Unlock() + if f.StreamID >= neverSent { + // We never asked for this. + cc.logf("http2: Transport received unsolicited DATA frame; closing connection") + return ConnectionError(ErrCodeProtocol) + } + // We probably did ask for this, but canceled. Just ignore it. + // TODO: be stricter here? only silently ignore things which + // we canceled, but not things which were closed normally + // by the peer? Tough without accumulating too much state. + return nil + } + if data := f.Data(); len(data) > 0 { + if cs.bufPipe.b == nil { + // Data frame after it's already closed? + cc.logf("http2: Transport received DATA frame for closed stream; closing connection") + return ConnectionError(ErrCodeProtocol) + } + + // Check connection-level flow control. + cc.mu.Lock() + if cs.inflow.available() >= int32(len(data)) { + cs.inflow.take(int32(len(data))) + } else { + cc.mu.Unlock() + return ConnectionError(ErrCodeFlowControl) + } + cc.mu.Unlock() + + if _, err := cs.bufPipe.Write(data); err != nil { + return err + } + } + + if f.StreamEnded() { + rl.endStream(cs) + } + return nil +} + +var errInvalidTrailers = errors.New("http2: invalid trailers") + +func (rl *clientConnReadLoop) endStream(cs *clientStream) { + // TODO: check that any declared content-length matches, like + // server.go's (*stream).endStream method. + err := io.EOF + code := cs.copyTrailers + if rl.reqMalformed != nil { + err = rl.reqMalformed + code = nil + } + cs.bufPipe.closeWithErrorAndCode(err, code) + delete(rl.activeRes, cs.ID) + if cs.req.Close || cs.req.Header.Get("Connection") == "close" { + rl.closeWhenIdle = true + } +} + +func (cs *clientStream) copyTrailers() { + for k, vv := range cs.trailer { + t := cs.resTrailer + if *t == nil { + *t = make(http.Header) + } + (*t)[k] = vv + } +} + +func (rl *clientConnReadLoop) processGoAway(f *GoAwayFrame) error { + cc := rl.cc + cc.t.connPool().MarkDead(cc) + if f.ErrCode != 0 { + // TODO: deal with GOAWAY more. particularly the error code + cc.vlogf("transport got GOAWAY with error code = %v", f.ErrCode) + } + cc.setGoAway(f) + return nil +} + +func (rl *clientConnReadLoop) processSettings(f *SettingsFrame) error { + cc := rl.cc + cc.mu.Lock() + defer cc.mu.Unlock() + return f.ForeachSetting(func(s Setting) error { + switch s.ID { + case SettingMaxFrameSize: + cc.maxFrameSize = s.Val + case SettingMaxConcurrentStreams: + cc.maxConcurrentStreams = s.Val + case SettingInitialWindowSize: + // TODO: error if this is too large. + + // TODO: adjust flow control of still-open + // frames by the difference of the old initial + // window size and this one. + cc.initialWindowSize = s.Val + default: + // TODO(bradfitz): handle more settings? SETTINGS_HEADER_TABLE_SIZE probably. + cc.vlogf("Unhandled Setting: %v", s) + } + return nil + }) +} + +func (rl *clientConnReadLoop) processWindowUpdate(f *WindowUpdateFrame) error { + cc := rl.cc + cs := cc.streamByID(f.StreamID, false) + if f.StreamID != 0 && cs == nil { + return nil + } + + cc.mu.Lock() + defer cc.mu.Unlock() + + fl := &cc.flow + if cs != nil { + fl = &cs.flow + } + if !fl.add(int32(f.Increment)) { + return ConnectionError(ErrCodeFlowControl) + } + cc.cond.Broadcast() + return nil +} + +func (rl *clientConnReadLoop) processResetStream(f *RSTStreamFrame) error { + cs := rl.cc.streamByID(f.StreamID, true) + if cs == nil { + // TODO: return error if server tries to RST_STEAM an idle stream + return nil + } + select { + case <-cs.peerReset: + // Already reset. + // This is the only goroutine + // which closes this, so there + // isn't a race. + default: + err := StreamError{cs.ID, f.ErrCode} + cs.resetErr = err + close(cs.peerReset) + cs.bufPipe.CloseWithError(err) + cs.cc.cond.Broadcast() // wake up checkReset via clientStream.awaitFlowControl + } + delete(rl.activeRes, cs.ID) + return nil +} + +func (rl *clientConnReadLoop) processPing(f *PingFrame) error { + if f.IsAck() { + // 6.7 PING: " An endpoint MUST NOT respond to PING frames + // containing this flag." + return nil + } + cc := rl.cc + cc.wmu.Lock() + defer cc.wmu.Unlock() + if err := cc.fr.WritePing(true, f.Data); err != nil { + return err + } + return cc.bw.Flush() +} + +func (rl *clientConnReadLoop) processPushPromise(f *PushPromiseFrame) error { + // We told the peer we don't want them. + // Spec says: + // "PUSH_PROMISE MUST NOT be sent if the SETTINGS_ENABLE_PUSH + // setting of the peer endpoint is set to 0. An endpoint that + // has set this setting and has received acknowledgement MUST + // treat the receipt of a PUSH_PROMISE frame as a connection + // error (Section 5.4.1) of type PROTOCOL_ERROR." + return ConnectionError(ErrCodeProtocol) +} + +func (cc *ClientConn) writeStreamReset(streamID uint32, code ErrCode, err error) { + // TODO: do something with err? send it as a debug frame to the peer? + // But that's only in GOAWAY. Invent a new frame type? Is there one already? + cc.wmu.Lock() + cc.fr.WriteRSTStream(streamID, code) + cc.bw.Flush() + cc.wmu.Unlock() +} + +var ( + errResponseHeaderListSize = errors.New("http2: response header list larger than advertised limit") + errPseudoTrailers = errors.New("http2: invalid pseudo header in trailers") +) + +func (rl *clientConnReadLoop) checkHeaderField(f hpack.HeaderField) bool { + if rl.reqMalformed != nil { + return false + } + + const headerFieldOverhead = 32 // per spec + rl.headerListSize += int64(len(f.Name)) + int64(len(f.Value)) + headerFieldOverhead + if max := rl.cc.t.maxHeaderListSize(); max != 0 && rl.headerListSize > int64(max) { + rl.hdec.SetEmitEnabled(false) + rl.reqMalformed = errResponseHeaderListSize + return false + } + + if !validHeaderFieldValue(f.Value) { + rl.reqMalformed = errInvalidHeaderFieldValue + return false + } + + isPseudo := strings.HasPrefix(f.Name, ":") + if isPseudo { + if rl.sawRegHeader { + rl.reqMalformed = errors.New("http2: invalid pseudo header after regular header") + return false + } + } else { + if !validHeaderFieldName(f.Name) { + rl.reqMalformed = errInvalidHeaderFieldName + return false + } + rl.sawRegHeader = true + } + + return true +} + +// onNewHeaderField runs on the readLoop goroutine whenever a new +// hpack header field is decoded. +func (rl *clientConnReadLoop) onNewHeaderField(f hpack.HeaderField) { + cc := rl.cc + if VerboseLogs { + cc.logf("http2: Transport decoded %v", f) + } + + if !rl.checkHeaderField(f) { + return + } + + isPseudo := strings.HasPrefix(f.Name, ":") + if isPseudo { + switch f.Name { + case ":status": + code, err := strconv.Atoi(f.Value) + if err != nil { + rl.reqMalformed = errors.New("http2: invalid :status") + return + } + rl.nextRes.Status = f.Value + " " + http.StatusText(code) + rl.nextRes.StatusCode = code + default: + // "Endpoints MUST NOT generate pseudo-header + // fields other than those defined in this + // document." + rl.reqMalformed = fmt.Errorf("http2: unknown response pseudo header %q", f.Name) + } + return + } + + key := http.CanonicalHeaderKey(f.Name) + if key == "Trailer" { + t := rl.nextRes.Trailer + if t == nil { + t = make(http.Header) + rl.nextRes.Trailer = t + } + foreachHeaderElement(f.Value, func(v string) { + t[http.CanonicalHeaderKey(v)] = nil + }) + } else { + rl.nextRes.Header.Add(key, f.Value) + } +} + +func (rl *clientConnReadLoop) onNewTrailerField(cs *clientStream, f hpack.HeaderField) { + if VerboseLogs { + rl.cc.logf("http2: Transport decoded trailer %v", f) + } + if !rl.checkHeaderField(f) { return } if strings.HasPrefix(f.Name, ":") { - // "Endpoints MUST NOT generate pseudo-header fields other than those defined in this document." - // TODO: treat as invalid? + // Pseudo-header fields MUST NOT appear in + // trailers. Endpoints MUST treat a request or + // response that contains undefined or invalid + // pseudo-header fields as malformed. + rl.reqMalformed = errPseudoTrailers return } - cc.nextRes.Header.Add(http.CanonicalHeaderKey(f.Name), f.Value) + + key := http.CanonicalHeaderKey(f.Name) + + // The spec says one must predeclare their trailers but in practice + // popular users (which is to say the only user we found) do not so we + // violate the spec and accept all of them. + const acceptAllTrailers = true + if _, ok := (*cs.resTrailer)[key]; ok || acceptAllTrailers { + if cs.trailer == nil { + cs.trailer = make(http.Header) + } + cs.trailer[key] = append(cs.trailer[key], f.Value) + } } + +func (cc *ClientConn) logf(format string, args ...interface{}) { + cc.t.logf(format, args...) +} + +func (cc *ClientConn) vlogf(format string, args ...interface{}) { + cc.t.vlogf(format, args...) +} + +func (t *Transport) vlogf(format string, args ...interface{}) { + if VerboseLogs { + t.logf(format, args...) + } +} + +func (t *Transport) logf(format string, args ...interface{}) { + log.Printf(format, args...) +} + +var noBody io.ReadCloser = ioutil.NopCloser(bytes.NewReader(nil)) + +func strSliceContains(ss []string, s string) bool { + for _, v := range ss { + if v == s { + return true + } + } + return false +} + +type erringRoundTripper struct{ err error } + +func (rt erringRoundTripper) RoundTrip(*http.Request) (*http.Response, error) { return nil, rt.err } + +// gzipReader wraps a response body so it can lazily +// call gzip.NewReader on the first call to Read +type gzipReader struct { + body io.ReadCloser // underlying Response.Body + zr *gzip.Reader // lazily-initialized gzip reader + zerr error // sticky error +} + +func (gz *gzipReader) Read(p []byte) (n int, err error) { + if gz.zerr != nil { + return 0, gz.zerr + } + if gz.zr == nil { + gz.zr, err = gzip.NewReader(gz.body) + if err != nil { + gz.zerr = err + return 0, err + } + } + return gz.zr.Read(p) +} + +func (gz *gzipReader) Close() error { + return gz.body.Close() +} + +type errorReader struct{ err error } + +func (r errorReader) Read(p []byte) (int, error) { return 0, r.err } diff --git a/vendor/golang.org/x/net/http2/write.go b/vendor/golang.org/x/net/http2/write.go index 02f0743de63..5297a4bfe3d 100644 --- a/vendor/golang.org/x/net/http2/write.go +++ b/vendor/golang.org/x/net/http2/write.go @@ -1,16 +1,15 @@ // Copyright 2014 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// See https://code.google.com/p/go/source/browse/CONTRIBUTORS -// Licensed under the same terms as Go itself: -// https://code.google.com/p/go/source/browse/LICENSE package http2 import ( "bytes" "fmt" + "log" "net/http" + "sort" "time" "golang.org/x/net/http2/hpack" @@ -26,7 +25,11 @@ type writeFramer interface { // frame writing scheduler (see writeScheduler in writesched.go). // // This interface is implemented by *serverConn. -// TODO: use it from the client code too, once it exists. +// +// TODO: decide whether to a) use this in the client code (which didn't +// end up using this yet, because it has a simpler design, not +// currently implementing priorities), or b) delete this and +// make the server code a bit more concrete. type writeContext interface { Framer() *Framer Flush() error @@ -44,6 +47,11 @@ func endsStream(w writeFramer) bool { return v.endStream case *writeResHeaders: return v.endStream + case nil: + // This can only happen if the caller reuses w after it's + // been intentionally nil'ed out to prevent use. Keep this + // here to catch future refactoring breaking it. + panic("endsStream called on nil writeFramer") } return false } @@ -89,6 +97,16 @@ func (w *writeData) writeFrame(ctx writeContext) error { return ctx.Framer().WriteData(w.streamID, w.endStream, w.p) } +// handlerPanicRST is the message sent from handler goroutines when +// the handler panics. +type handlerPanicRST struct { + StreamID uint32 +} + +func (hp handlerPanicRST) writeFrame(ctx writeContext) error { + return ctx.Framer().WriteRSTStream(hp.StreamID, ErrCodeInternal) +} + func (se StreamError) writeFrame(ctx writeContext) error { return ctx.Framer().WriteRSTStream(se.StreamID, se.Code) } @@ -106,40 +124,48 @@ func (writeSettingsAck) writeFrame(ctx writeContext) error { } // writeResHeaders is a request to write a HEADERS and 0+ CONTINUATION frames -// for HTTP response headers from a server handler. +// for HTTP response headers or trailers from a server handler. type writeResHeaders struct { streamID uint32 - httpResCode int + httpResCode int // 0 means no ":status" line h http.Header // may be nil + trailers []string // if non-nil, which keys of h to write. nil means all. endStream bool + date string contentType string contentLength string } +func encKV(enc *hpack.Encoder, k, v string) { + if VerboseLogs { + log.Printf("http2: server encoding header %q = %q", k, v) + } + enc.WriteField(hpack.HeaderField{Name: k, Value: v}) +} + func (w *writeResHeaders) writeFrame(ctx writeContext) error { enc, buf := ctx.HeaderEncoder() buf.Reset() - enc.WriteField(hpack.HeaderField{Name: ":status", Value: httpCodeString(w.httpResCode)}) - for k, vv := range w.h { - k = lowerHeader(k) - for _, v := range vv { - // TODO: more of "8.1.2.2 Connection-Specific Header Fields" - if k == "transfer-encoding" && v != "trailers" { - continue - } - enc.WriteField(hpack.HeaderField{Name: k, Value: v}) - } + + if w.httpResCode != 0 { + encKV(enc, ":status", httpCodeString(w.httpResCode)) } + + encodeHeaders(enc, w.h, w.trailers) + if w.contentType != "" { - enc.WriteField(hpack.HeaderField{Name: "content-type", Value: w.contentType}) + encKV(enc, "content-type", w.contentType) } if w.contentLength != "" { - enc.WriteField(hpack.HeaderField{Name: "content-length", Value: w.contentLength}) + encKV(enc, "content-length", w.contentLength) + } + if w.date != "" { + encKV(enc, "date", w.date) } headerBlock := buf.Bytes() - if len(headerBlock) == 0 { + if len(headerBlock) == 0 && w.trailers == nil { panic("unexpected empty hpack") } @@ -185,7 +211,7 @@ type write100ContinueHeadersFrame struct { func (w write100ContinueHeadersFrame) writeFrame(ctx writeContext) error { enc, buf := ctx.HeaderEncoder() buf.Reset() - enc.WriteField(hpack.HeaderField{Name: ":status", Value: "100"}) + encKV(enc, ":status", "100") return ctx.Framer().WriteHeaders(HeadersFrameParam{ StreamID: w.streamID, BlockFragment: buf.Bytes(), @@ -202,3 +228,36 @@ type writeWindowUpdate struct { func (wu writeWindowUpdate) writeFrame(ctx writeContext) error { return ctx.Framer().WriteWindowUpdate(wu.streamID, wu.n) } + +func encodeHeaders(enc *hpack.Encoder, h http.Header, keys []string) { + // TODO: garbage. pool sorters like http1? hot path for 1 key? + if keys == nil { + keys = make([]string, 0, len(h)) + for k := range h { + keys = append(keys, k) + } + sort.Strings(keys) + } + for _, k := range keys { + vv := h[k] + k = lowerHeader(k) + if !validHeaderFieldName(k) { + // TODO: return an error? golang.org/issue/14048 + // For now just omit it. + continue + } + isTE := k == "transfer-encoding" + for _, v := range vv { + if !validHeaderFieldValue(v) { + // TODO: return an error? golang.org/issue/14048 + // For now just omit it. + continue + } + // TODO: more of "8.1.2.2 Connection-Specific Header Fields" + if isTE && v != "trailers" { + continue + } + encKV(enc, k, v) + } + } +} diff --git a/vendor/golang.org/x/net/http2/writesched.go b/vendor/golang.org/x/net/http2/writesched.go index 0e1b7486fbb..c24316ce7b2 100644 --- a/vendor/golang.org/x/net/http2/writesched.go +++ b/vendor/golang.org/x/net/http2/writesched.go @@ -1,9 +1,6 @@ // Copyright 2014 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// See https://code.google.com/p/go/source/browse/CONTRIBUTORS -// Licensed under the same terms as Go itself: -// https://code.google.com/p/go/source/browse/LICENSE package http2 diff --git a/vendor/golang.org/x/net/trace/trace.go b/vendor/golang.org/x/net/trace/trace.go index c87290b76eb..197402e038f 100644 --- a/vendor/golang.org/x/net/trace/trace.go +++ b/vendor/golang.org/x/net/trace/trace.go @@ -113,6 +113,7 @@ func init() { http.Error(w, "not allowed", http.StatusUnauthorized) return } + w.Header().Set("Content-Type", "text/html; charset=utf-8") Render(w, req, sensitive) }) http.HandleFunc("/debug/events", func(w http.ResponseWriter, req *http.Request) { @@ -121,6 +122,7 @@ func init() { http.Error(w, "not allowed", http.StatusUnauthorized) return } + w.Header().Set("Content-Type", "text/html; charset=utf-8") RenderEvents(w, req, sensitive) }) } @@ -172,7 +174,7 @@ func Render(w io.Writer, req *http.Request, sensitive bool) { completedMu.RLock() data.Families = make([]string, 0, len(completedTraces)) - for fam, _ := range completedTraces { + for fam := range completedTraces { data.Families = append(data.Families, fam) } completedMu.RUnlock()