mirror of
https://github.com/kubeshark/kubeshark.git
synced 2025-08-06 19:14:50 +00:00
Remove github.com/up9inc/mizu/tap/extensions/http/lib
This commit is contained in:
M. Mert Yildiran
parent
2166801910
commit
e1167c4065
@ -1,7 +1,3 @@
|
|||||||
module github.com/up9inc/mizu/tap/extensions/http
|
module github.com/up9inc/mizu/tap/extensions/http
|
||||||
|
|
||||||
go 1.16
|
go 1.16
|
||||||
|
|
||||||
require github.com/up9inc/mizu/tap/extensions/http/lib v0.0.0-00010101000000-000000000000
|
|
||||||
|
|
||||||
replace github.com/up9inc/mizu/tap/extensions/http/lib => ./lib
|
|
||||||
|
|||||||
@ -1,11 +0,0 @@
|
|||||||
module github.com/up9inc/mizu/tap/extensions/http/lib
|
|
||||||
|
|
||||||
go 1.16
|
|
||||||
|
|
||||||
require (
|
|
||||||
github.com/bradleyfalzon/tlsx v0.0.0-20170624122154-28fd0e59bac4
|
|
||||||
github.com/google/martian v2.1.0+incompatible
|
|
||||||
github.com/romana/rlog v0.0.0-20171115192701-f018bc92e7d7
|
|
||||||
golang.org/x/net v0.0.0-20210421230115-4e50805a0758
|
|
||||||
golang.org/x/text v0.3.7 // indirect
|
|
||||||
)
|
|
||||||
@ -1,17 +0,0 @@
|
|||||||
github.com/bradleyfalzon/tlsx v0.0.0-20170624122154-28fd0e59bac4 h1:NJOOlc6ZJjix0A1rAU+nxruZtR8KboG1848yqpIUo4M=
|
|
||||||
github.com/bradleyfalzon/tlsx v0.0.0-20170624122154-28fd0e59bac4/go.mod h1:DQPxZS994Ld1Y8uwnJT+dRL04XPD0cElP/pHH/zEBHM=
|
|
||||||
github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
|
|
||||||
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
|
|
||||||
github.com/orcaman/concurrent-map v0.0.0-20210501183033-44dafcb38ecc h1:Ak86L+yDSOzKFa7WM5bf5itSOo1e3Xh8bm5YCMUXIjQ=
|
|
||||||
github.com/orcaman/concurrent-map v0.0.0-20210501183033-44dafcb38ecc/go.mod h1:Lu3tH6HLW3feq74c2GC+jIMS/K2CFcDWnWD9XkenwhI=
|
|
||||||
github.com/romana/rlog v0.0.0-20171115192701-f018bc92e7d7 h1:jkvpcEatpwuMF5O5LVxTnehj6YZ/aEZN4NWD/Xml4pI=
|
|
||||||
github.com/romana/rlog v0.0.0-20171115192701-f018bc92e7d7/go.mod h1:KTrHyWpO1sevuXPZwyeZc72ddWRFqNSKDFl7uVWKpg0=
|
|
||||||
golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d h1:LO7XpTYMwTqxjLcGWPijK3vRXg1aWdlNOVOHRq45d7c=
|
|
||||||
golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
|
||||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
|
||||||
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
|
||||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
|
||||||
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
|
||||||
golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
|
|
||||||
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
|
||||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
|
||||||
@ -1,308 +0,0 @@
|
|||||||
package lib
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"io/ioutil"
|
|
||||||
"log"
|
|
||||||
"net/http"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/google/martian/har"
|
|
||||||
"github.com/romana/rlog"
|
|
||||||
)
|
|
||||||
|
|
||||||
var outputLevel int
|
|
||||||
var errorsMap map[string]uint
|
|
||||||
var errorsMapMutex sync.Mutex
|
|
||||||
var nErrors uint
|
|
||||||
var ownIps []string // global
|
|
||||||
var hostMode bool // global
|
|
||||||
|
|
||||||
func logError(minOutputLevel int, t string, s string, a ...interface{}) {
|
|
||||||
errorsMapMutex.Lock()
|
|
||||||
nErrors++
|
|
||||||
nb, _ := errorsMap[t]
|
|
||||||
errorsMap[t] = nb + 1
|
|
||||||
errorsMapMutex.Unlock()
|
|
||||||
|
|
||||||
if outputLevel >= minOutputLevel {
|
|
||||||
formatStr := fmt.Sprintf("%s: %s", t, s)
|
|
||||||
rlog.Errorf(formatStr, a...)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
func Error(t string, s string, a ...interface{}) {
|
|
||||||
logError(0, t, s, a...)
|
|
||||||
}
|
|
||||||
func SilentError(t string, s string, a ...interface{}) {
|
|
||||||
logError(2, t, s, a...)
|
|
||||||
}
|
|
||||||
func Debug(s string, a ...interface{}) {
|
|
||||||
rlog.Debugf(s, a...)
|
|
||||||
}
|
|
||||||
func Trace(s string, a ...interface{}) {
|
|
||||||
rlog.Tracef(1, s, a...)
|
|
||||||
}
|
|
||||||
|
|
||||||
const readPermission = 0644
|
|
||||||
const harFilenameSuffix = ".har"
|
|
||||||
const tempFilenameSuffix = ".har.tmp"
|
|
||||||
|
|
||||||
type PairChanItem struct {
|
|
||||||
Request *http.Request
|
|
||||||
RequestTime time.Time
|
|
||||||
Response *http.Response
|
|
||||||
ResponseTime time.Time
|
|
||||||
RequestSenderIp string
|
|
||||||
ConnectionInfo *ConnectionInfo
|
|
||||||
}
|
|
||||||
|
|
||||||
func openNewHarFile(filename string) *HarFile {
|
|
||||||
file, err := os.OpenFile(filename, os.O_APPEND|os.O_CREATE|os.O_WRONLY, readPermission)
|
|
||||||
if err != nil {
|
|
||||||
log.Panicf("Failed to open output file: %s (%v,%+v)", err, err, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
harFile := HarFile{file: file, entryCount: 0}
|
|
||||||
harFile.writeHeader()
|
|
||||||
|
|
||||||
return &harFile
|
|
||||||
}
|
|
||||||
|
|
||||||
type HarFile struct {
|
|
||||||
file *os.File
|
|
||||||
entryCount int
|
|
||||||
}
|
|
||||||
|
|
||||||
func NewEntry(request *http.Request, requestTime time.Time, response *http.Response, responseTime time.Time) (*har.Entry, error) {
|
|
||||||
harRequest, err := har.NewRequest(request, false)
|
|
||||||
if err != nil {
|
|
||||||
SilentError("convert-request-to-har", "Failed converting request to HAR %s (%v,%+v)", err, err, err)
|
|
||||||
return nil, errors.New("Failed converting request to HAR")
|
|
||||||
}
|
|
||||||
|
|
||||||
// For requests with multipart/form-data or application/x-www-form-urlencoded Content-Type,
|
|
||||||
// martian/har will parse the request body and place the parameters in harRequest.PostData.Params
|
|
||||||
// instead of harRequest.PostData.Text (as the HAR spec requires it).
|
|
||||||
// Mizu currently only looks at PostData.Text. Therefore, instead of letting martian/har set the content of
|
|
||||||
// PostData, always copy the request body to PostData.Text.
|
|
||||||
if request.ContentLength > 0 {
|
|
||||||
reqBody, err := ioutil.ReadAll(request.Body)
|
|
||||||
if err != nil {
|
|
||||||
SilentError("read-request-body", "Failed converting request to HAR %s (%v,%+v)", err, err, err)
|
|
||||||
return nil, errors.New("Failed reading request body")
|
|
||||||
}
|
|
||||||
request.Body = ioutil.NopCloser(bytes.NewReader(reqBody))
|
|
||||||
harRequest.PostData.Text = string(reqBody)
|
|
||||||
}
|
|
||||||
|
|
||||||
harResponse, err := har.NewResponse(response, true)
|
|
||||||
if err != nil {
|
|
||||||
SilentError("convert-response-to-har", "Failed converting response to HAR %s (%v,%+v)", err, err, err)
|
|
||||||
return nil, errors.New("Failed converting response to HAR")
|
|
||||||
}
|
|
||||||
|
|
||||||
if harRequest.PostData != nil && strings.HasPrefix(harRequest.PostData.MimeType, "application/grpc") {
|
|
||||||
// Force HTTP/2 gRPC into HAR template
|
|
||||||
|
|
||||||
harRequest.URL = fmt.Sprintf("%s://%s%s", request.Header.Get(":scheme"), request.Header.Get(":authority"), request.Header.Get(":path"))
|
|
||||||
|
|
||||||
status, err := strconv.Atoi(response.Header.Get(":status"))
|
|
||||||
if err != nil {
|
|
||||||
SilentError("convert-response-status-for-har", "Failed converting status to int %s (%v,%+v)", err, err, err)
|
|
||||||
return nil, errors.New("Failed converting response status to int for HAR")
|
|
||||||
}
|
|
||||||
harResponse.Status = status
|
|
||||||
} else {
|
|
||||||
// Martian copies http.Request.URL.String() to har.Request.URL, which usually contains the path.
|
|
||||||
// However, according to the HAR spec, the URL field needs to be the absolute URL.
|
|
||||||
var scheme string
|
|
||||||
if request.URL.Scheme != "" {
|
|
||||||
scheme = request.URL.Scheme
|
|
||||||
} else {
|
|
||||||
scheme = "http"
|
|
||||||
}
|
|
||||||
harRequest.URL = fmt.Sprintf("%s://%s%s", scheme, request.Host, request.URL)
|
|
||||||
}
|
|
||||||
|
|
||||||
totalTime := responseTime.Sub(requestTime).Round(time.Millisecond).Milliseconds()
|
|
||||||
if totalTime < 1 {
|
|
||||||
totalTime = 1
|
|
||||||
}
|
|
||||||
|
|
||||||
harEntry := har.Entry{
|
|
||||||
StartedDateTime: time.Now().UTC(),
|
|
||||||
Time: totalTime,
|
|
||||||
Request: harRequest,
|
|
||||||
Response: harResponse,
|
|
||||||
Cache: &har.Cache{},
|
|
||||||
Timings: &har.Timings{
|
|
||||||
Send: -1,
|
|
||||||
Wait: -1,
|
|
||||||
Receive: totalTime,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
return &harEntry, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f *HarFile) WriteEntry(harEntry *har.Entry) {
|
|
||||||
harEntryJson, err := json.Marshal(harEntry)
|
|
||||||
if err != nil {
|
|
||||||
SilentError("har-entry-marshal", "Failed converting har entry object to JSON%s (%v,%+v)", err, err, err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
var separator string
|
|
||||||
if f.GetEntryCount() > 0 {
|
|
||||||
separator = ","
|
|
||||||
} else {
|
|
||||||
separator = ""
|
|
||||||
}
|
|
||||||
|
|
||||||
harEntryString := append([]byte(separator), harEntryJson...)
|
|
||||||
|
|
||||||
if _, err := f.file.Write(harEntryString); err != nil {
|
|
||||||
log.Panicf("Failed to write to output file: %s (%v,%+v)", err, err, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
f.entryCount++
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f *HarFile) GetEntryCount() int {
|
|
||||||
return f.entryCount
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f *HarFile) Close() {
|
|
||||||
f.writeTrailer()
|
|
||||||
|
|
||||||
err := f.file.Close()
|
|
||||||
if err != nil {
|
|
||||||
log.Panicf("Failed to close output file: %s (%v,%+v)", err, err, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f *HarFile) writeHeader() {
|
|
||||||
header := []byte(`{"log": {"version": "1.2", "creator": {"name": "Mizu", "version": "0.0.1"}, "entries": [`)
|
|
||||||
if _, err := f.file.Write(header); err != nil {
|
|
||||||
log.Panicf("Failed to write header to output file: %s (%v,%+v)", err, err, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f *HarFile) writeTrailer() {
|
|
||||||
trailer := []byte("]}}")
|
|
||||||
if _, err := f.file.Write(trailer); err != nil {
|
|
||||||
log.Panicf("Failed to write trailer to output file: %s (%v,%+v)", err, err, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func NewHarWriter(outputDir string, maxEntries int) *HarWriter {
|
|
||||||
return &HarWriter{
|
|
||||||
OutputDirPath: outputDir,
|
|
||||||
MaxEntries: maxEntries,
|
|
||||||
PairChan: make(chan *PairChanItem),
|
|
||||||
OutChan: make(chan *OutputChannelItem, 1000),
|
|
||||||
currentFile: nil,
|
|
||||||
done: make(chan bool),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
type OutputChannelItem struct {
|
|
||||||
HarEntry *har.Entry
|
|
||||||
ConnectionInfo *ConnectionInfo
|
|
||||||
ValidationRulesChecker string
|
|
||||||
}
|
|
||||||
|
|
||||||
type HarWriter struct {
|
|
||||||
OutputDirPath string
|
|
||||||
MaxEntries int
|
|
||||||
PairChan chan *PairChanItem
|
|
||||||
OutChan chan *OutputChannelItem
|
|
||||||
currentFile *HarFile
|
|
||||||
done chan bool
|
|
||||||
}
|
|
||||||
|
|
||||||
func (hw *HarWriter) WritePair(request *http.Request, requestTime time.Time, response *http.Response, responseTime time.Time, connectionInfo *ConnectionInfo) {
|
|
||||||
hw.PairChan <- &PairChanItem{
|
|
||||||
Request: request,
|
|
||||||
RequestTime: requestTime,
|
|
||||||
Response: response,
|
|
||||||
ResponseTime: responseTime,
|
|
||||||
ConnectionInfo: connectionInfo,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (hw *HarWriter) Start() {
|
|
||||||
if hw.OutputDirPath != "" {
|
|
||||||
if err := os.MkdirAll(hw.OutputDirPath, os.ModePerm); err != nil {
|
|
||||||
log.Panicf("Failed to create output directory: %s (%v,%+v)", err, err, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
go func() {
|
|
||||||
for pair := range hw.PairChan {
|
|
||||||
harEntry, err := NewEntry(pair.Request, pair.RequestTime, pair.Response, pair.ResponseTime)
|
|
||||||
if err != nil {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
if hw.OutputDirPath != "" {
|
|
||||||
if hw.currentFile == nil {
|
|
||||||
hw.openNewFile()
|
|
||||||
}
|
|
||||||
|
|
||||||
hw.currentFile.WriteEntry(harEntry)
|
|
||||||
|
|
||||||
if hw.currentFile.GetEntryCount() >= hw.MaxEntries {
|
|
||||||
hw.closeFile()
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
hw.OutChan <- &OutputChannelItem{
|
|
||||||
HarEntry: harEntry,
|
|
||||||
ConnectionInfo: pair.ConnectionInfo,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if hw.currentFile != nil {
|
|
||||||
hw.closeFile()
|
|
||||||
}
|
|
||||||
hw.done <- true
|
|
||||||
}()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (hw *HarWriter) Stop() {
|
|
||||||
close(hw.PairChan)
|
|
||||||
<-hw.done
|
|
||||||
close(hw.OutChan)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (hw *HarWriter) openNewFile() {
|
|
||||||
filename := buildFilename(hw.OutputDirPath, time.Now(), tempFilenameSuffix)
|
|
||||||
hw.currentFile = openNewHarFile(filename)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (hw *HarWriter) closeFile() {
|
|
||||||
hw.currentFile.Close()
|
|
||||||
tmpFilename := hw.currentFile.file.Name()
|
|
||||||
hw.currentFile = nil
|
|
||||||
|
|
||||||
filename := buildFilename(hw.OutputDirPath, time.Now(), harFilenameSuffix)
|
|
||||||
err := os.Rename(tmpFilename, filename)
|
|
||||||
if err != nil {
|
|
||||||
SilentError("Rename-file", "cannot rename file: %s (%v,%+v)", err, err, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func buildFilename(dir string, t time.Time, suffix string) string {
|
|
||||||
// (epoch time in nanoseconds)__(YYYY_Month_DD__hh-mm-ss).har
|
|
||||||
filename := fmt.Sprintf("%d__%s%s", t.UnixNano(), t.Format("2006_Jan_02__15-04-05"), suffix)
|
|
||||||
return filepath.Join(dir, filename)
|
|
||||||
}
|
|
||||||
@ -1,116 +0,0 @@
|
|||||||
package lib
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"net/http"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
var reqResMatcher = createResponseRequestMatcher() // global
|
|
||||||
|
|
||||||
type requestResponsePair struct {
|
|
||||||
Request httpMessage `json:"request"`
|
|
||||||
Response httpMessage `json:"response"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type httpMessage struct {
|
|
||||||
isRequest bool
|
|
||||||
captureTime time.Time
|
|
||||||
orig interface{}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Key is {client_addr}:{client_port}->{dest_addr}:{dest_port}
|
|
||||||
type requestResponseMatcher struct {
|
|
||||||
openMessagesMap sync.Map
|
|
||||||
}
|
|
||||||
|
|
||||||
func createResponseRequestMatcher() requestResponseMatcher {
|
|
||||||
newMatcher := &requestResponseMatcher{openMessagesMap: sync.Map{}}
|
|
||||||
return *newMatcher
|
|
||||||
}
|
|
||||||
|
|
||||||
func (matcher *requestResponseMatcher) registerRequest(ident string, request *http.Request, captureTime time.Time) *requestResponsePair {
|
|
||||||
split := splitIdent(ident)
|
|
||||||
key := genKey(split)
|
|
||||||
|
|
||||||
requestHTTPMessage := httpMessage{
|
|
||||||
isRequest: true,
|
|
||||||
captureTime: captureTime,
|
|
||||||
orig: request,
|
|
||||||
}
|
|
||||||
|
|
||||||
if response, found := matcher.openMessagesMap.LoadAndDelete(key); found {
|
|
||||||
// Type assertion always succeeds because all of the map's values are of httpMessage type
|
|
||||||
responseHTTPMessage := response.(*httpMessage)
|
|
||||||
if responseHTTPMessage.isRequest {
|
|
||||||
SilentError("Request-Duplicate", "Got duplicate request with same identifier")
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
Trace("Matched open Response for %s", key)
|
|
||||||
return matcher.preparePair(&requestHTTPMessage, responseHTTPMessage)
|
|
||||||
}
|
|
||||||
|
|
||||||
matcher.openMessagesMap.Store(key, &requestHTTPMessage)
|
|
||||||
Trace("Registered open Request for %s", key)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (matcher *requestResponseMatcher) registerResponse(ident string, response *http.Response, captureTime time.Time) *requestResponsePair {
|
|
||||||
split := splitIdent(ident)
|
|
||||||
key := genKey(split)
|
|
||||||
|
|
||||||
responseHTTPMessage := httpMessage{
|
|
||||||
isRequest: false,
|
|
||||||
captureTime: captureTime,
|
|
||||||
orig: response,
|
|
||||||
}
|
|
||||||
|
|
||||||
if request, found := matcher.openMessagesMap.LoadAndDelete(key); found {
|
|
||||||
// Type assertion always succeeds because all of the map's values are of httpMessage type
|
|
||||||
requestHTTPMessage := request.(*httpMessage)
|
|
||||||
if !requestHTTPMessage.isRequest {
|
|
||||||
SilentError("Response-Duplicate", "Got duplicate response with same identifier")
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
Trace("Matched open Request for %s", key)
|
|
||||||
return matcher.preparePair(requestHTTPMessage, &responseHTTPMessage)
|
|
||||||
}
|
|
||||||
|
|
||||||
matcher.openMessagesMap.Store(key, &responseHTTPMessage)
|
|
||||||
Trace("Registered open Response for %s", key)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (matcher *requestResponseMatcher) preparePair(requestHTTPMessage *httpMessage, responseHTTPMessage *httpMessage) *requestResponsePair {
|
|
||||||
return &requestResponsePair{
|
|
||||||
Request: *requestHTTPMessage,
|
|
||||||
Response: *responseHTTPMessage,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func splitIdent(ident string) []string {
|
|
||||||
ident = strings.Replace(ident, "->", " ", -1)
|
|
||||||
return strings.Split(ident, " ")
|
|
||||||
}
|
|
||||||
|
|
||||||
func genKey(split []string) string {
|
|
||||||
key := fmt.Sprintf("%s:%s->%s:%s,%s", split[0], split[2], split[1], split[3], split[4])
|
|
||||||
return key
|
|
||||||
}
|
|
||||||
|
|
||||||
func (matcher *requestResponseMatcher) deleteOlderThan(t time.Time) int {
|
|
||||||
numDeleted := 0
|
|
||||||
|
|
||||||
matcher.openMessagesMap.Range(func(key interface{}, value interface{}) bool {
|
|
||||||
message, _ := value.(*httpMessage)
|
|
||||||
if message.captureTime.Before(t) {
|
|
||||||
matcher.openMessagesMap.Delete(key)
|
|
||||||
numDeleted++
|
|
||||||
}
|
|
||||||
return true
|
|
||||||
})
|
|
||||||
|
|
||||||
return numDeleted
|
|
||||||
}
|
|
||||||
@ -1,39 +0,0 @@
|
|||||||
package lib
|
|
||||||
|
|
||||||
type OutboundLinkProtocol string
|
|
||||||
|
|
||||||
const (
|
|
||||||
TLSProtocol OutboundLinkProtocol = "tls"
|
|
||||||
)
|
|
||||||
|
|
||||||
type OutboundLink struct {
|
|
||||||
Src string
|
|
||||||
DstIP string
|
|
||||||
DstPort int
|
|
||||||
SuggestedResolvedName string
|
|
||||||
SuggestedProtocol OutboundLinkProtocol
|
|
||||||
}
|
|
||||||
|
|
||||||
func NewOutboundLinkWriter() *OutboundLinkWriter {
|
|
||||||
return &OutboundLinkWriter{
|
|
||||||
OutChan: make(chan *OutboundLink),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
type OutboundLinkWriter struct {
|
|
||||||
OutChan chan *OutboundLink
|
|
||||||
}
|
|
||||||
|
|
||||||
func (olw *OutboundLinkWriter) WriteOutboundLink(src string, DstIP string, DstPort int, SuggestedResolvedName string, SuggestedProtocol OutboundLinkProtocol) {
|
|
||||||
olw.OutChan <- &OutboundLink{
|
|
||||||
Src: src,
|
|
||||||
DstIP: DstIP,
|
|
||||||
DstPort: DstPort,
|
|
||||||
SuggestedResolvedName: SuggestedResolvedName,
|
|
||||||
SuggestedProtocol: SuggestedProtocol,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (olw *OutboundLinkWriter) Stop() {
|
|
||||||
close(olw.OutChan)
|
|
||||||
}
|
|
||||||
@ -1,307 +0,0 @@
|
|||||||
package lib
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bufio"
|
|
||||||
"bytes"
|
|
||||||
"encoding/hex"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"io/ioutil"
|
|
||||||
"net/http"
|
|
||||||
"strconv"
|
|
||||||
"sync"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/bradleyfalzon/tlsx"
|
|
||||||
)
|
|
||||||
|
|
||||||
const checkTLSPacketAmount = 100
|
|
||||||
|
|
||||||
type httpReaderDataMsg struct {
|
|
||||||
bytes []byte
|
|
||||||
timestamp time.Time
|
|
||||||
}
|
|
||||||
|
|
||||||
type tcpID struct {
|
|
||||||
srcIP string
|
|
||||||
dstIP string
|
|
||||||
srcPort string
|
|
||||||
dstPort string
|
|
||||||
}
|
|
||||||
|
|
||||||
type ConnectionInfo struct {
|
|
||||||
ClientIP string
|
|
||||||
ClientPort string
|
|
||||||
ServerIP string
|
|
||||||
ServerPort string
|
|
||||||
IsOutgoing bool
|
|
||||||
}
|
|
||||||
|
|
||||||
func SomeExample() {
|
|
||||||
fmt.Println("Hello from SomeExample")
|
|
||||||
}
|
|
||||||
|
|
||||||
func (tid *tcpID) String() string {
|
|
||||||
return fmt.Sprintf("%s->%s %s->%s", tid.srcIP, tid.dstIP, tid.srcPort, tid.dstPort)
|
|
||||||
}
|
|
||||||
|
|
||||||
/* httpReader gets reads from a channel of bytes of tcp payload, and parses it into HTTP/1 requests and responses.
|
|
||||||
* The payload is written to the channel by a tap.TcpStream object that is dedicated to one tcp connection.
|
|
||||||
* An httpReader object is unidirectional: it parses either a client stream or a server stream.
|
|
||||||
* Implements io.Reader interface (Read)
|
|
||||||
*/
|
|
||||||
type httpReader struct {
|
|
||||||
ident string
|
|
||||||
tcpID tcpID
|
|
||||||
isClient bool
|
|
||||||
isHTTP2 bool
|
|
||||||
isOutgoing bool
|
|
||||||
msgQueue chan httpReaderDataMsg // Channel of captured reassembled tcp payload
|
|
||||||
data []byte
|
|
||||||
captureTime time.Time
|
|
||||||
hexdump bool
|
|
||||||
messageCount uint
|
|
||||||
harWriter *HarWriter
|
|
||||||
packetsSeen uint
|
|
||||||
outboundLinkWriter *OutboundLinkWriter
|
|
||||||
}
|
|
||||||
|
|
||||||
func (h *httpReader) Read(p []byte) (int, error) {
|
|
||||||
var msg httpReaderDataMsg
|
|
||||||
|
|
||||||
ok := true
|
|
||||||
for ok && len(h.data) == 0 {
|
|
||||||
msg, ok = <-h.msgQueue
|
|
||||||
h.data = msg.bytes
|
|
||||||
|
|
||||||
h.captureTime = msg.timestamp
|
|
||||||
if len(h.data) > 0 {
|
|
||||||
h.packetsSeen += 1
|
|
||||||
}
|
|
||||||
if h.packetsSeen < checkTLSPacketAmount && len(msg.bytes) > 5 { // packets with less than 5 bytes cause tlsx to panic
|
|
||||||
clientHello := tlsx.ClientHello{}
|
|
||||||
err := clientHello.Unmarshall(msg.bytes)
|
|
||||||
if err == nil {
|
|
||||||
fmt.Printf("Detected TLS client hello with SNI %s\n", clientHello.SNI)
|
|
||||||
numericPort, _ := strconv.Atoi(h.tcpID.dstPort)
|
|
||||||
h.outboundLinkWriter.WriteOutboundLink(h.tcpID.srcIP, h.tcpID.dstIP, numericPort, clientHello.SNI, TLSProtocol)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if !ok || len(h.data) == 0 {
|
|
||||||
return 0, io.EOF
|
|
||||||
}
|
|
||||||
|
|
||||||
l := copy(p, h.data)
|
|
||||||
h.data = h.data[l:]
|
|
||||||
return l, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (h *httpReader) run(wg *sync.WaitGroup) {
|
|
||||||
defer wg.Done()
|
|
||||||
b := bufio.NewReader(h)
|
|
||||||
|
|
||||||
// if isHTTP2, err := checkIsHTTP2Connection(b, h.isClient); err != nil {
|
|
||||||
// SilentError("HTTP/2-Prepare-Connection", "stream %s Failed to check if client is HTTP/2: %s (%v,%+v)", h.ident, err, err, err)
|
|
||||||
// // Do something?
|
|
||||||
// } else {
|
|
||||||
// h.isHTTP2 = isHTTP2
|
|
||||||
// }
|
|
||||||
|
|
||||||
// if h.isHTTP2 {
|
|
||||||
// err := prepareHTTP2Connection(b, h.isClient)
|
|
||||||
// if err != nil {
|
|
||||||
// SilentError("HTTP/2-Prepare-Connection-After-Check", "stream %s error: %s (%v,%+v)", h.ident, err, err, err)
|
|
||||||
// }
|
|
||||||
// h.grpcAssembler = createGrpcAssembler(b)
|
|
||||||
// }
|
|
||||||
|
|
||||||
for true {
|
|
||||||
if h.isHTTP2 {
|
|
||||||
err := h.handleHTTP2Stream()
|
|
||||||
if err == io.EOF || err == io.ErrUnexpectedEOF {
|
|
||||||
break
|
|
||||||
} else if err != nil {
|
|
||||||
SilentError("HTTP/2", "stream %s error: %s (%v,%+v)", h.ident, err, err, err)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
} else if h.isClient {
|
|
||||||
err := h.handleHTTP1ClientStream(b)
|
|
||||||
if err == io.EOF || err == io.ErrUnexpectedEOF {
|
|
||||||
break
|
|
||||||
} else if err != nil {
|
|
||||||
SilentError("HTTP-request", "stream %s Request error: %s (%v,%+v)", h.ident, err, err, err)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
err := h.handleHTTP1ServerStream(b)
|
|
||||||
if err == io.EOF || err == io.ErrUnexpectedEOF {
|
|
||||||
break
|
|
||||||
} else if err != nil {
|
|
||||||
SilentError("HTTP-response", "stream %s Response error: %s (%v,%+v)", h.ident, err, err, err)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (h *httpReader) handleHTTP2Stream() error {
|
|
||||||
// streamID, messageHTTP1, err := h.grpcAssembler.readMessage()
|
|
||||||
// h.messageCount++
|
|
||||||
// if err != nil {
|
|
||||||
// return err
|
|
||||||
// }
|
|
||||||
|
|
||||||
// var reqResPair *requestResponsePair
|
|
||||||
// var connectionInfo *ConnectionInfo
|
|
||||||
|
|
||||||
// switch messageHTTP1 := messageHTTP1.(type) {
|
|
||||||
// case http.Request:
|
|
||||||
// ident := fmt.Sprintf("%s->%s %s->%s %d", h.tcpID.srcIP, h.tcpID.dstIP, h.tcpID.srcPort, h.tcpID.dstPort, streamID)
|
|
||||||
// connectionInfo = &ConnectionInfo{
|
|
||||||
// ClientIP: h.tcpID.srcIP,
|
|
||||||
// ClientPort: h.tcpID.srcPort,
|
|
||||||
// ServerIP: h.tcpID.dstIP,
|
|
||||||
// ServerPort: h.tcpID.dstPort,
|
|
||||||
// IsOutgoing: h.isOutgoing,
|
|
||||||
// }
|
|
||||||
// reqResPair = reqResMatcher.registerRequest(ident, &messageHTTP1, h.captureTime)
|
|
||||||
// case http.Response:
|
|
||||||
// ident := fmt.Sprintf("%s->%s %s->%s %d", h.tcpID.dstIP, h.tcpID.srcIP, h.tcpID.dstPort, h.tcpID.srcPort, streamID)
|
|
||||||
// connectionInfo = &ConnectionInfo{
|
|
||||||
// ClientIP: h.tcpID.dstIP,
|
|
||||||
// ClientPort: h.tcpID.dstPort,
|
|
||||||
// ServerIP: h.tcpID.srcIP,
|
|
||||||
// ServerPort: h.tcpID.srcPort,
|
|
||||||
// IsOutgoing: h.isOutgoing,
|
|
||||||
// }
|
|
||||||
// reqResPair = reqResMatcher.registerResponse(ident, &messageHTTP1, h.captureTime)
|
|
||||||
// }
|
|
||||||
|
|
||||||
// if reqResPair != nil {
|
|
||||||
// // statsTracker.incMatchedMessages()
|
|
||||||
|
|
||||||
// if h.harWriter != nil {
|
|
||||||
// h.harWriter.WritePair(
|
|
||||||
// reqResPair.Request.orig.(*http.Request),
|
|
||||||
// reqResPair.Request.captureTime,
|
|
||||||
// reqResPair.Response.orig.(*http.Response),
|
|
||||||
// reqResPair.Response.captureTime,
|
|
||||||
// connectionInfo,
|
|
||||||
// )
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (h *httpReader) handleHTTP1ClientStream(b *bufio.Reader) error {
|
|
||||||
req, err := http.ReadRequest(b)
|
|
||||||
h.messageCount++
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
body, err := ioutil.ReadAll(req.Body)
|
|
||||||
req.Body = io.NopCloser(bytes.NewBuffer(body)) // rewind
|
|
||||||
s := len(body)
|
|
||||||
if err != nil {
|
|
||||||
SilentError("HTTP-request-body", "stream %s Got body err: %s", h.ident, err)
|
|
||||||
} else if h.hexdump {
|
|
||||||
Debug("Body(%d/0x%x) - %s", len(body), len(body), hex.Dump(body))
|
|
||||||
}
|
|
||||||
if err := req.Body.Close(); err != nil {
|
|
||||||
SilentError("HTTP-request-body-close", "stream %s Failed to close request body: %s", h.ident, err)
|
|
||||||
}
|
|
||||||
encoding := req.Header["Content-Encoding"]
|
|
||||||
Debug("HTTP/1 Request: %s %s %s (Body:%d) -> %s", h.ident, req.Method, req.URL, s, encoding)
|
|
||||||
|
|
||||||
ident := fmt.Sprintf("%s->%s %s->%s %d", h.tcpID.srcIP, h.tcpID.dstIP, h.tcpID.srcPort, h.tcpID.dstPort, h.messageCount)
|
|
||||||
reqResPair := reqResMatcher.registerRequest(ident, req, h.captureTime)
|
|
||||||
if reqResPair != nil {
|
|
||||||
// statsTracker.incMatchedMessages()
|
|
||||||
|
|
||||||
if h.harWriter != nil {
|
|
||||||
h.harWriter.WritePair(
|
|
||||||
reqResPair.Request.orig.(*http.Request),
|
|
||||||
reqResPair.Request.captureTime,
|
|
||||||
reqResPair.Response.orig.(*http.Response),
|
|
||||||
reqResPair.Response.captureTime,
|
|
||||||
&ConnectionInfo{
|
|
||||||
ClientIP: h.tcpID.srcIP,
|
|
||||||
ClientPort: h.tcpID.srcPort,
|
|
||||||
ServerIP: h.tcpID.dstIP,
|
|
||||||
ServerPort: h.tcpID.dstPort,
|
|
||||||
IsOutgoing: h.isOutgoing,
|
|
||||||
},
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// h.parent.Lock()
|
|
||||||
// h.parent.urls = append(h.parent.urls, req.URL.String())
|
|
||||||
// h.parent.Unlock()
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (h *httpReader) handleHTTP1ServerStream(b *bufio.Reader) error {
|
|
||||||
res, err := http.ReadResponse(b, nil)
|
|
||||||
h.messageCount++
|
|
||||||
var req string
|
|
||||||
// h.parent.Lock()
|
|
||||||
// if len(h.parent.urls) == 0 {
|
|
||||||
// req = fmt.Sprintf("<no-request-seen>")
|
|
||||||
// } else {
|
|
||||||
// req, h.parent.urls = h.parent.urls[0], h.parent.urls[1:]
|
|
||||||
// }
|
|
||||||
// h.parent.Unlock()
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
body, err := ioutil.ReadAll(res.Body)
|
|
||||||
res.Body = io.NopCloser(bytes.NewBuffer(body)) // rewind
|
|
||||||
s := len(body)
|
|
||||||
if err != nil {
|
|
||||||
SilentError("HTTP-response-body", "HTTP/%s: failed to get body(parsed len:%d): %s", h.ident, s, err)
|
|
||||||
}
|
|
||||||
if h.hexdump {
|
|
||||||
Debug("Body(%d/0x%x) - %s", len(body), len(body), hex.Dump(body))
|
|
||||||
}
|
|
||||||
if err := res.Body.Close(); err != nil {
|
|
||||||
SilentError("HTTP-response-body-close", "HTTP/%s: failed to close body(parsed len:%d): %s", h.ident, s, err)
|
|
||||||
}
|
|
||||||
sym := ","
|
|
||||||
if res.ContentLength > 0 && res.ContentLength != int64(s) {
|
|
||||||
sym = "!="
|
|
||||||
}
|
|
||||||
contentType, ok := res.Header["Content-Type"]
|
|
||||||
if !ok {
|
|
||||||
contentType = []string{http.DetectContentType(body)}
|
|
||||||
}
|
|
||||||
encoding := res.Header["Content-Encoding"]
|
|
||||||
Debug("HTTP/1 Response: %s %s URL:%s (%d%s%d%s) -> %s", h.ident, res.Status, req, res.ContentLength, sym, s, contentType, encoding)
|
|
||||||
|
|
||||||
ident := fmt.Sprintf("%s->%s %s->%s %d", h.tcpID.dstIP, h.tcpID.srcIP, h.tcpID.dstPort, h.tcpID.srcPort, h.messageCount)
|
|
||||||
reqResPair := reqResMatcher.registerResponse(ident, res, h.captureTime)
|
|
||||||
if reqResPair != nil {
|
|
||||||
// statsTracker.incMatchedMessages()
|
|
||||||
|
|
||||||
if h.harWriter != nil {
|
|
||||||
h.harWriter.WritePair(
|
|
||||||
reqResPair.Request.orig.(*http.Request),
|
|
||||||
reqResPair.Request.captureTime,
|
|
||||||
reqResPair.Response.orig.(*http.Response),
|
|
||||||
reqResPair.Response.captureTime,
|
|
||||||
&ConnectionInfo{
|
|
||||||
ClientIP: h.tcpID.dstIP,
|
|
||||||
ClientPort: h.tcpID.dstPort,
|
|
||||||
ServerIP: h.tcpID.srcIP,
|
|
||||||
ServerPort: h.tcpID.srcPort,
|
|
||||||
IsOutgoing: h.isOutgoing,
|
|
||||||
},
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
@ -2,15 +2,12 @@ package main
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
|
|
||||||
"github.com/up9inc/mizu/tap/extensions/http/lib"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
type greeting string
|
type greeting string
|
||||||
|
|
||||||
func (g greeting) Greet() {
|
func (g greeting) Greet() {
|
||||||
fmt.Println("Hello HTTP")
|
fmt.Println("Hello HTTP")
|
||||||
lib.SomeExample()
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// exported as symbol named "Greeter"
|
// exported as symbol named "Greeter"
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user