diff --git a/agent/go.mod b/agent/go.mod index a8d77083d..2d1012eb3 100644 --- a/agent/go.mod +++ b/agent/go.mod @@ -17,7 +17,6 @@ require ( github.com/romana/rlog v0.0.0-20171115192701-f018bc92e7d7 github.com/up9inc/mizu/shared v0.0.0 github.com/up9inc/mizu/tap v0.0.0 - github.com/up9inc/mizu/tap/api v0.0.0 github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0 gorm.io/driver/sqlite v1.1.4 gorm.io/gorm v1.21.8 diff --git a/agent/go.sum b/agent/go.sum index 4d92b6ed2..12760e2dd 100644 --- a/agent/go.sum +++ b/agent/go.sum @@ -42,8 +42,6 @@ github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb0 github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/bradleyfalzon/tlsx v0.0.0-20170624122154-28fd0e59bac4 h1:NJOOlc6ZJjix0A1rAU+nxruZtR8KboG1848yqpIUo4M= -github.com/bradleyfalzon/tlsx v0.0.0-20170624122154-28fd0e59bac4/go.mod h1:DQPxZS994Ld1Y8uwnJT+dRL04XPD0cElP/pHH/zEBHM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= @@ -298,9 +296,8 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210224082022-3d97a244fca7 h1:OgUuv8lsRpBibGNbSizVwKWlysjaNzmC9gYMhPVfqFM= golang.org/x/net v0.0.0-20210224082022-3d97a244fca7/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210421230115-4e50805a0758 h1:aEpZnXcAmXkd6AvLb2OPt+EN1Zu/8Ne3pCqPjja5PXY= -golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -340,9 +337,8 @@ golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073 h1:8qxJSnu+7dRq6upnbntrmriWByIakBuct5OM/MdQC1M= golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe h1:WdX7u8s3yOigWAhHEaDl8r9G+4XwFQEQFtBMYyN+kXQ= -golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d h1:SZxvLBoTP5yHO3Frd4z4vrF+DBX9vMVanchswa69toE= @@ -352,9 +348,8 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= -golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= diff --git a/tap/extensions/http/go.mod b/tap/extensions/http/go.mod index 6274daec5..d1dc80f1a 100644 --- a/tap/extensions/http/go.mod +++ b/tap/extensions/http/go.mod @@ -3,9 +3,14 @@ module github.com/up9inc/mizu/tap/extensions/http go 1.16 require ( - github.com/google/martian v2.1.0+incompatible // indirect - github.com/romana/rlog v0.0.0-20171115192701-f018bc92e7d7 // indirect + github.com/google/martian v2.1.0+incompatible + github.com/romana/rlog v0.0.0-20171115192701-f018bc92e7d7 github.com/up9inc/mizu/tap/api v0.0.0 + golang.org/x/net v0.0.0-20210224082022-3d97a244fca7 + golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073 + golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d + golang.org/x/text v0.3.4 + golang.org/x/tools v0.0.0-20210106214847-113979e3529a ) replace github.com/up9inc/mizu/tap/api v0.0.0 => ../../api diff --git a/tap/extensions/http/go.sum b/tap/extensions/http/go.sum index f25f09c37..d99c45d08 100644 --- a/tap/extensions/http/go.sum +++ b/tap/extensions/http/go.sum @@ -2,3 +2,36 @@ github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPg github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/romana/rlog v0.0.0-20171115192701-f018bc92e7d7 h1:jkvpcEatpwuMF5O5LVxTnehj6YZ/aEZN4NWD/Xml4pI= github.com/romana/rlog v0.0.0-20171115192701-f018bc92e7d7/go.mod h1:KTrHyWpO1sevuXPZwyeZc72ddWRFqNSKDFl7uVWKpg0= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210224082022-3d97a244fca7 h1:OgUuv8lsRpBibGNbSizVwKWlysjaNzmC9gYMhPVfqFM= +golang.org/x/net v0.0.0-20210224082022-3d97a244fca7/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073 h1:8qxJSnu+7dRq6upnbntrmriWByIakBuct5OM/MdQC1M= +golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d h1:SZxvLBoTP5yHO3Frd4z4vrF+DBX9vMVanchswa69toE= +golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a h1:CB3a9Nez8M13wwlr/E2YtwoU+qYHKfC+JrDa45RXXoQ= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/tap/extensions/http/grpc_assembler.go b/tap/extensions/http/grpc_assembler.go new file mode 100644 index 000000000..94dec2692 --- /dev/null +++ b/tap/extensions/http/grpc_assembler.go @@ -0,0 +1,243 @@ +package main + +import ( + "bufio" + "bytes" + "encoding/base64" + "encoding/binary" + "errors" + "io" + "math" + "net/http" + "net/url" + "strings" + + "golang.org/x/net/http2" + "golang.org/x/net/http2/hpack" +) + +const frameHeaderLen = 9 + +var clientPreface = []byte(http2.ClientPreface) + +const initialHeaderTableSize = 4096 +const protoHTTP2 = "HTTP/2.0" +const protoMajorHTTP2 = 2 +const protoMinorHTTP2 = 0 + +var maxHTTP2DataLen int = 1 * 1024 * 1024 // 1MB + +type messageFragment struct { + headers []hpack.HeaderField + data []byte +} + +type fragmentsByStream map[uint32]*messageFragment + +func (fbs *fragmentsByStream) appendFrame(streamID uint32, frame http2.Frame) { + switch frame := frame.(type) { + case *http2.MetaHeadersFrame: + if existingFragment, ok := (*fbs)[streamID]; ok { + existingFragment.headers = append(existingFragment.headers, frame.Fields...) + } else { + // new fragment + (*fbs)[streamID] = &messageFragment{headers: frame.Fields} + } + case *http2.DataFrame: + newDataLen := len(frame.Data()) + if existingFragment, ok := (*fbs)[streamID]; ok { + existingDataLen := len(existingFragment.data) + // Never save more than maxHTTP2DataLen bytes + numBytesToAppend := int(math.Min(float64(maxHTTP2DataLen-existingDataLen), float64(newDataLen))) + + existingFragment.data = append(existingFragment.data, frame.Data()[:numBytesToAppend]...) + } else { + // new fragment + // In principle, should not happen with DATA frames, because they are always preceded by HEADERS + + // Never save more than maxHTTP2DataLen bytes + numBytesToAppend := int(math.Min(float64(maxHTTP2DataLen), float64(newDataLen))) + + (*fbs)[streamID] = &messageFragment{data: frame.Data()[:numBytesToAppend]} + } + } +} + +func (fbs *fragmentsByStream) pop(streamID uint32) ([]hpack.HeaderField, []byte) { + headers := (*fbs)[streamID].headers + data := (*fbs)[streamID].data + delete(*fbs, streamID) + + return headers, data +} + +func createGrpcAssembler(b *bufio.Reader) *GrpcAssembler { + var framerOutput bytes.Buffer + framer := http2.NewFramer(&framerOutput, b) + framer.ReadMetaHeaders = hpack.NewDecoder(initialHeaderTableSize, nil) + return &GrpcAssembler{ + fragmentsByStream: make(fragmentsByStream), + framer: framer, + } +} + +type GrpcAssembler struct { + fragmentsByStream fragmentsByStream + framer *http2.Framer +} + +func (ga *GrpcAssembler) readMessage() (uint32, interface{}, error) { + // Exactly one Framer is used for each half connection. + // (Instead of creating a new Framer for each ReadFrame operation) + // This is needed in order to decompress the headers, + // because the compression context is updated with each requests/response. + frame, err := ga.framer.ReadFrame() + if err != nil { + return 0, nil, err + } + + streamID := frame.Header().StreamID + + ga.fragmentsByStream.appendFrame(streamID, frame) + + if !(ga.isStreamEnd(frame)) { + return 0, nil, nil + } + + headers, data := ga.fragmentsByStream.pop(streamID) + + // Note: header keys are converted by http.Header.Set to canonical names, e.g. content-type -> Content-Type. + // By converting the keys we violate the HTTP/2 specification, which state that all headers must be lowercase. + headersHTTP1 := make(http.Header) + for _, header := range headers { + headersHTTP1.Add(header.Name, header.Value) + } + dataString := base64.StdEncoding.EncodeToString(data) + + // Use http1 types only because they are expected in http_matcher. + // TODO: Create an interface that will be used by http_matcher:registerRequest and http_matcher:registerRequest + // to accept both HTTP/1.x and HTTP/2 requests and responses + var messageHTTP1 interface{} + if _, ok := headersHTTP1[":method"]; ok { + messageHTTP1 = http.Request{ + URL: &url.URL{}, + Method: "POST", + Header: headersHTTP1, + Proto: protoHTTP2, + ProtoMajor: protoMajorHTTP2, + ProtoMinor: protoMinorHTTP2, + Body: io.NopCloser(strings.NewReader(dataString)), + ContentLength: int64(len(dataString)), + } + } else if _, ok := headersHTTP1[":status"]; ok { + messageHTTP1 = http.Response{ + Header: headersHTTP1, + Proto: protoHTTP2, + ProtoMajor: protoMajorHTTP2, + ProtoMinor: protoMinorHTTP2, + Body: io.NopCloser(strings.NewReader(dataString)), + ContentLength: int64(len(dataString)), + } + } else { + return 0, nil, errors.New("Failed to assemble stream: neither a request nor a message") + } + + return streamID, messageHTTP1, nil +} + +func (ga *GrpcAssembler) isStreamEnd(frame http2.Frame) bool { + switch frame := frame.(type) { + case *http2.MetaHeadersFrame: + if frame.StreamEnded() { + return true + } + case *http2.DataFrame: + if frame.StreamEnded() { + return true + } + } + + return false +} + +/* Check if HTTP/2. Remove HTTP/2 client preface from start of buffer if present + */ +func checkIsHTTP2Connection(b *bufio.Reader, isClient bool) (bool, error) { + if isClient { + return checkIsHTTP2ClientStream(b) + } + + return checkIsHTTP2ServerStream(b) +} + +func prepareHTTP2Connection(b *bufio.Reader, isClient bool) error { + if !isClient { + return nil + } + + return discardClientPreface(b) +} + +func checkIsHTTP2ClientStream(b *bufio.Reader) (bool, error) { + return checkClientPreface(b) +} + +func checkIsHTTP2ServerStream(b *bufio.Reader) (bool, error) { + buf, err := b.Peek(frameHeaderLen) + if err != nil { + return false, err + } + + // If response starts with this text, it is HTTP/1.x + if bytes.Compare(buf, []byte("HTTP/1.0 ")) == 0 || bytes.Compare(buf, []byte("HTTP/1.1 ")) == 0 { + return false, nil + } + + // Check server connection preface (a settings frame) + frameHeader := http2.FrameHeader{ + Length: uint32(buf[0])<<16 | uint32(buf[1])<<8 | uint32(buf[2]), + Type: http2.FrameType(buf[3]), + Flags: http2.Flags(buf[4]), + StreamID: binary.BigEndian.Uint32(buf[5:]) & (1<<31 - 1), + } + + if frameHeader.Type != http2.FrameSettings { + // If HTTP/2, but not start of stream, will also fulfill this condition. + return false, nil + } + + return true, nil +} + +func checkClientPreface(b *bufio.Reader) (bool, error) { + bytesStart, err := b.Peek(len(clientPreface)) + if err != nil { + return false, err + } else if len(bytesStart) != len(clientPreface) { + return false, errors.New("checkClientPreface: not enough bytes read") + } + + if !bytes.Equal(bytesStart, clientPreface) { + return false, nil + } + + return true, nil +} + +func discardClientPreface(b *bufio.Reader) error { + if isClientPrefacePresent, err := checkClientPreface(b); err != nil { + return err + } else if !isClientPrefacePresent { + return errors.New("discardClientPreface: does not begin with client preface") + } + + // Remove client preface string from the buffer + n, err := b.Discard(len(clientPreface)) + if err != nil { + return err + } else if n != len(clientPreface) { + return errors.New("discardClientPreface: failed to discard client preface") + } + + return nil +} diff --git a/tap/extensions/http/handlers.go b/tap/extensions/http/handlers.go new file mode 100644 index 000000000..132d3d654 --- /dev/null +++ b/tap/extensions/http/handlers.go @@ -0,0 +1,105 @@ +package main + +import ( + "bufio" + "fmt" + "io" + "io/ioutil" + "log" + "net/http" + "time" + + "github.com/up9inc/mizu/tap/api" +) + +func handleHTTP2Stream(grpcAssembler *GrpcAssembler, tcpID *api.TcpID) (*api.RequestResponsePair, error) { + streamID, messageHTTP1, err := grpcAssembler.readMessage() + if err != nil { + return nil, err + } + + var reqResPair *api.RequestResponsePair + + switch messageHTTP1 := messageHTTP1.(type) { + case http.Request: + requestCounter++ + ident := fmt.Sprintf( + "%s->%s %s->%s %d", + tcpID.SrcIP, + tcpID.DstIP, + tcpID.SrcPort, + tcpID.DstPort, + streamID, + ) + reqResPair = reqResMatcher.registerRequest(ident, &messageHTTP1, time.Now()) + case http.Response: + responseCounter++ + ident := fmt.Sprintf( + "%s->%s %s->%s %d", + tcpID.DstIP, + tcpID.SrcIP, + tcpID.DstPort, + tcpID.SrcPort, + streamID, + ) + reqResPair = reqResMatcher.registerResponse(ident, &messageHTTP1, time.Now()) + } + + if reqResPair != nil { + return reqResPair, nil + } + + return nil, nil +} + +func handleHTTP1ClientStream(b *bufio.Reader, tcpID *api.TcpID) error { + requestCounter++ + req, err := http.ReadRequest(b) + if err == io.EOF || err == io.ErrUnexpectedEOF { + return nil + } else if err != nil { + log.Println("Error reading stream:", err) + } else { + body, _ := ioutil.ReadAll(req.Body) + req.Body.Close() + log.Printf("Received request: %+v with body: %+v\n", req, body) + } + + ident := fmt.Sprintf( + "%s->%s %s->%s %d", + tcpID.SrcIP, + tcpID.DstIP, + tcpID.SrcPort, + tcpID.DstPort, + requestCounter, + ) + reqResMatcher.registerRequest(ident, req, time.Now()) + return err +} + +func handleHTTP1ServerStream(b *bufio.Reader, tcpID *api.TcpID) (*api.RequestResponsePair, error) { + responseCounter++ + res, err := http.ReadResponse(b, nil) + if err == io.EOF || err == io.ErrUnexpectedEOF { + return nil, nil + } else if err != nil { + log.Println("Error reading stream:", err) + } else { + body, _ := ioutil.ReadAll(res.Body) + res.Body.Close() + log.Printf("Received response: %+v with body: %+v\n", res, body) + } + ident := fmt.Sprintf( + "%s->%s %s->%s %d", + tcpID.DstIP, + tcpID.SrcIP, + tcpID.DstPort, + tcpID.SrcPort, + responseCounter, + ) + reqResPair := reqResMatcher.registerResponse(ident, res, time.Now()) + if reqResPair != nil { + return reqResPair, nil + } + return nil, err +} diff --git a/tap/extensions/http/main.go b/tap/extensions/http/main.go index 9f6e5f2d7..f463ed604 100644 --- a/tap/extensions/http/main.go +++ b/tap/extensions/http/main.go @@ -4,10 +4,7 @@ import ( "bufio" "fmt" "io" - "io/ioutil" "log" - "net/http" - "time" "github.com/up9inc/mizu/tap/api" ) @@ -34,53 +31,53 @@ func (g dissecting) Ping() { } func (g dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID) *api.RequestResponsePair { - for { - if isClient { - requestCounter++ - req, err := http.ReadRequest(b) - if err == io.EOF || err == io.ErrUnexpectedEOF { - return nil - } else if err != nil { - log.Println("Error reading stream:", err) - } else { - body, _ := ioutil.ReadAll(req.Body) - req.Body.Close() - log.Printf("Received request: %+v with body: %+v\n", req, body) - } + ident := fmt.Sprintf("%s->%s:%s->%s", tcpID.SrcIP, tcpID.DstIP, tcpID.SrcPort, tcpID.DstPort) + isHTTP2, err := checkIsHTTP2Connection(b, isClient) + if err != nil { + SilentError("HTTP/2-Prepare-Connection", "stream %s Failed to check if client is HTTP/2: %s (%v,%+v)", ident, err, err, err) + // Do something? + } - ident := fmt.Sprintf( - "%s->%s %s->%s %d", - tcpID.SrcIP, - tcpID.DstIP, - tcpID.SrcPort, - tcpID.DstPort, - requestCounter, - ) - reqResMatcher.registerRequest(ident, req, time.Now()) - } else { - responseCounter++ - res, err := http.ReadResponse(b, nil) - if err == io.EOF || err == io.ErrUnexpectedEOF { - return nil - } else if err != nil { - log.Println("Error reading stream:", err) - } else { - body, _ := ioutil.ReadAll(res.Body) - res.Body.Close() - log.Printf("Received response: %+v with body: %+v\n", res, body) - } - ident := fmt.Sprintf( - "%s->%s %s->%s %d", - tcpID.DstIP, - tcpID.SrcIP, - tcpID.DstPort, - tcpID.SrcPort, - responseCounter, - ) - reqResPair := reqResMatcher.registerResponse(ident, res, time.Now()) + var grpcAssembler *GrpcAssembler + if isHTTP2 { + err := prepareHTTP2Connection(b, isClient) + if err != nil { + SilentError("HTTP/2-Prepare-Connection-After-Check", "stream %s error: %s (%v,%+v)", ident, err, err, err) + } + grpcAssembler = createGrpcAssembler(b) + } + + for { + if isHTTP2 { + reqResPair, err := handleHTTP2Stream(grpcAssembler, tcpID) if reqResPair != nil { return reqResPair } + if err == io.EOF || err == io.ErrUnexpectedEOF { + break + } else if err != nil { + SilentError("HTTP/2", "stream %s error: %s (%v,%+v)", ident, err, err, err) + continue + } + } else if isClient { + err := handleHTTP1ClientStream(b, tcpID) + if err == io.EOF || err == io.ErrUnexpectedEOF { + break + } else if err != nil { + SilentError("HTTP-request", "stream %s Request error: %s (%v,%+v)", ident, err, err, err) + continue + } + } else { + reqResPair, err := handleHTTP1ServerStream(b, tcpID) + if reqResPair != nil { + return reqResPair + } + if err == io.EOF || err == io.ErrUnexpectedEOF { + break + } else if err != nil { + SilentError("HTTP-response", "stream %s Response error: %s (%v,%+v)", ident, err, err, err) + continue + } } } return nil diff --git a/tap/go.mod b/tap/go.mod index 4b7dbd388..0df13d898 100644 --- a/tap/go.mod +++ b/tap/go.mod @@ -3,11 +3,14 @@ module github.com/up9inc/mizu/tap go 1.16 require ( - github.com/bradleyfalzon/tlsx v0.0.0-20170624122154-28fd0e59bac4 // indirect github.com/google/gopacket v1.1.19 github.com/romana/rlog v0.0.0-20171115192701-f018bc92e7d7 github.com/up9inc/mizu/tap/api v0.0.0 - golang.org/x/net v0.0.0-20210421230115-4e50805a0758 // indirect + golang.org/x/net v0.0.0-20210224082022-3d97a244fca7 + golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073 + golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d + golang.org/x/text v0.3.4 + golang.org/x/tools v0.0.0-20210106214847-113979e3529a ) replace github.com/up9inc/mizu/tap/api v0.0.0 => ./api diff --git a/tap/go.sum b/tap/go.sum index 6cce30cd2..b852c21d6 100644 --- a/tap/go.sum +++ b/tap/go.sum @@ -1,26 +1,41 @@ -github.com/bradleyfalzon/tlsx v0.0.0-20170624122154-28fd0e59bac4 h1:NJOOlc6ZJjix0A1rAU+nxruZtR8KboG1848yqpIUo4M= -github.com/bradleyfalzon/tlsx v0.0.0-20170624122154-28fd0e59bac4/go.mod h1:DQPxZS994Ld1Y8uwnJT+dRL04XPD0cElP/pHH/zEBHM= github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8= github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo= github.com/romana/rlog v0.0.0-20171115192701-f018bc92e7d7 h1:jkvpcEatpwuMF5O5LVxTnehj6YZ/aEZN4NWD/Xml4pI= github.com/romana/rlog v0.0.0-20171115192701-f018bc92e7d7/go.mod h1:KTrHyWpO1sevuXPZwyeZc72ddWRFqNSKDFl7uVWKpg0= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20210421230115-4e50805a0758 h1:aEpZnXcAmXkd6AvLb2OPt+EN1Zu/8Ne3pCqPjja5PXY= -golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210224082022-3d97a244fca7 h1:OgUuv8lsRpBibGNbSizVwKWlysjaNzmC9gYMhPVfqFM= +golang.org/x/net v0.0.0-20210224082022-3d97a244fca7/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5/tI9ujCIVX+P5KiHuI= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe h1:WdX7u8s3yOigWAhHEaDl8r9G+4XwFQEQFtBMYyN+kXQ= -golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073 h1:8qxJSnu+7dRq6upnbntrmriWByIakBuct5OM/MdQC1M= +golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d h1:SZxvLBoTP5yHO3Frd4z4vrF+DBX9vMVanchswa69toE= +golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a h1:CB3a9Nez8M13wwlr/E2YtwoU+qYHKfC+JrDa45RXXoQ= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=