github/kubeshark
1
0
Fork 0
mirror of https://github.com/kubeshark/kubeshark.git synced 2025-08-02 09:07:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

Move TrafficFilteringOptions struct to shared module

Browse Source
This commit is contained in:
M. Mert Yildiran 2022-04-20 12:13:27 +03:00
parent 110325598a
commit 960ba644cd
No known key found for this signature in database
GPG Key ID: D42ADB236521BF7A
31 changed files with 4535 additions and 72 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
agent/main.go
View File

@ -251,17 +251,17 @@ func setUIFlags(uiIndexPath string) error {
return nil
}
func getTrafficFilteringOptions() *tapApi.TrafficFilteringOptions {
func getTrafficFilteringOptions() *shared.TrafficFilteringOptions {
filteringOptionsJson := os.Getenv(shared.MizuFilteringOptionsEnvVar)
if filteringOptionsJson == "" {
return &tapApi.TrafficFilteringOptions{
return &shared.TrafficFilteringOptions{
IgnoredUserAgents: []string{},
}
}
var filteringOptions tapApi.TrafficFilteringOptions
var filteringOptions shared.TrafficFilteringOptions
err := json.Unmarshal([]byte(filteringOptionsJson), &filteringOptions)
if err != nil {
panic(fmt.Sprintf("env var %s's value of %s is invalid! json must match the api.TrafficFilteringOptions struct %v", shared.MizuFilteringOptionsEnvVar, filteringOptionsJson, err))
panic(fmt.Sprintf("env var %s's value of %s is invalid! json must match the shared.TrafficFilteringOptions struct %v", shared.MizuFilteringOptionsEnvVar, filteringOptionsJson, err))
}
return &filteringOptions

6
cli/cmd/tapRunner.go
View File

@ -189,7 +189,7 @@ func printTappedPodsPreview(ctx context.Context, kubernetesProvider *kubernetes.
}
}
func startTapperSyncer(ctx context.Context, cancel context.CancelFunc, provider *kubernetes.Provider, targetNamespaces []string, mizuApiFilteringOptions api.TrafficFilteringOptions, startTime time.Time) error {
func startTapperSyncer(ctx context.Context, cancel context.CancelFunc, provider *kubernetes.Provider, targetNamespaces []string, mizuApiFilteringOptions shared.TrafficFilteringOptions, startTime time.Time) error {
tapperSyncer, err := kubernetes.CreateAndStartMizuTapperSyncer(ctx, provider, kubernetes.TapperSyncerConfig{
TargetNamespaces: targetNamespaces,
PodFilterRegex: *config.Config.Tap.PodRegex(),
@ -275,7 +275,7 @@ func readValidationRules(file string) (string, error) {
return string(newContent), nil
}
func getMizuApiFilteringOptions() (*api.TrafficFilteringOptions, error) {
func getMizuApiFilteringOptions() (*shared.TrafficFilteringOptions, error) {
var compiledRegexSlice []*api.SerializableRegexp
if config.Config.Tap.PlainTextFilterRegexes != nil && len(config.Config.Tap.PlainTextFilterRegexes) > 0 {
@ -289,7 +289,7 @@ func getMizuApiFilteringOptions() (*api.TrafficFilteringOptions, error) {
}
}
return &api.TrafficFilteringOptions{
return &shared.TrafficFilteringOptions{
PlainTextMaskingRegexes: compiledRegexSlice,
IgnoredUserAgents: config.Config.Tap.IgnoredUserAgents,
DisableRedaction: config.Config.Tap.DisableRedaction,

2
shared/go.mod
View File

@ -6,7 +6,6 @@ require (
github.com/docker/go-units v0.4.0
github.com/golang-jwt/jwt/v4 v4.2.0
github.com/op/go-logging v0.0.0-20160315200505-970db520ece7
github.com/up9inc/mizu/tap/api v0.0.0
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
k8s.io/api v0.23.3
k8s.io/apimachinery v0.23.3
@ -41,7 +40,6 @@ require (
github.com/google/btree v1.0.1 // indirect
github.com/google/go-cmp v0.5.7 // indirect
github.com/google/gofuzz v1.2.0 // indirect
github.com/google/martian v2.1.0+incompatible // indirect
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
github.com/google/uuid v1.3.0 // indirect
github.com/googleapis/gnostic v0.5.5 // indirect

1
shared/go.sum
View File

@ -282,7 +282,6 @@ github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/
github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=

3
shared/kubernetes/mizuTapperSyncer.go
View File

@ -10,7 +10,6 @@ import (
"github.com/up9inc/mizu/shared"
"github.com/up9inc/mizu/shared/debounce"
"github.com/up9inc/mizu/shared/logger"
"github.com/up9inc/mizu/tap/api"
core "k8s.io/api/core/v1"
)
@ -44,7 +43,7 @@ type TapperSyncerConfig struct {
ImagePullPolicy core.PullPolicy
LogLevel logging.Level
IgnoredUserAgents []string
MizuApiFilteringOptions api.TrafficFilteringOptions
MizuApiFilteringOptions shared.TrafficFilteringOptions
MizuServiceAccountExists bool
ServiceMesh bool
Tls bool

3
shared/kubernetes/provider.go
View File

@ -15,7 +15,6 @@ import (
"github.com/up9inc/mizu/shared"
"github.com/up9inc/mizu/shared/logger"
"github.com/up9inc/mizu/shared/semver"
"github.com/up9inc/mizu/tap/api"
auth "k8s.io/api/authorization/v1"
core "k8s.io/api/core/v1"
rbac "k8s.io/api/rbac/v1"
@ -708,7 +707,7 @@ func (provider *Provider) CreateConfigMap(ctx context.Context, namespace string,
return nil
}
func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespace string, daemonSetName string, podImage string, tapperPodName string, apiServerPodIp string, nodeNames []string, serviceAccountName string, resources shared.Resources, imagePullPolicy core.PullPolicy, mizuApiFilteringOptions api.TrafficFilteringOptions, logLevel logging.Level, serviceMesh bool, tls bool) error {
func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespace string, daemonSetName string, podImage string, tapperPodName string, apiServerPodIp string, nodeNames []string, serviceAccountName string, resources shared.Resources, imagePullPolicy core.PullPolicy, mizuApiFilteringOptions shared.TrafficFilteringOptions, logLevel logging.Level, serviceMesh bool, tls bool) error {
logger.Log.Debugf("Applying %d tapper daemon sets, ns: %s, daemonSetName: %s, podImage: %s, tapperPodName: %s", len(nodeNames), namespace, daemonSetName, podImage, tapperPodName)
if len(nodeNames) == 0 {

2
tap/api/options.go → shared/options.go
View File

@ -1,4 +1,4 @@
package api
package shared
type TrafficFilteringOptions struct {
IgnoredUserAgents []string

2
tap/api/serializable_regexp.go → shared/serializable_regexp.go
View File

@ -1,4 +1,4 @@
package api
package shared
import "regexp"

3
tap/api/api.go
View File

@ -15,6 +15,7 @@ import (
"time"
"github.com/google/martian/har"
"github.com/up9inc/mizu/shared"
"github.com/up9inc/mizu/tap/api/diagnose"
)
@ -131,7 +132,7 @@ func (p *ReadProgress) Current() (n int) {
type Dissector interface {
Register(*Extension)
Ping()
Dissect(b *bufio.Reader, progress *ReadProgress, capture Capture, isClient bool, tcpID *TcpID, counterPair *CounterPair, superTimer *SuperTimer, superIdentifier *SuperIdentifier, emitter Emitter, options *TrafficFilteringOptions, reqResMatcher RequestResponseMatcher) error
Dissect(b *bufio.Reader, progress *ReadProgress, capture Capture, isClient bool, tcpID *TcpID, counterPair *CounterPair, superTimer *SuperTimer, superIdentifier *SuperIdentifier, emitter Emitter, options *shared.TrafficFilteringOptions, reqResMatcher RequestResponseMatcher) error
Analyze(item *OutputChannelItem, resolvedSource string, resolvedDestination string, namespace string) *Entry
Summarize(entry *Entry) *BaseEntry
Represent(request map[string]interface{}, response map[string]interface{}) (object []byte, err error)

3
tap/api/tcp_reader.go
View File

@ -7,6 +7,7 @@ import (
"sync"
"time"
"github.com/up9inc/mizu/shared"
"github.com/up9inc/mizu/shared/logger"
)
@ -72,7 +73,7 @@ func (h *TcpReader) Close() {
h.Unlock()
}
func (h *TcpReader) Run(filteringOptions *TrafficFilteringOptions, wg *sync.WaitGroup) {
func (h *TcpReader) Run(filteringOptions *shared.TrafficFilteringOptions, wg *sync.WaitGroup) {
defer wg.Done()
b := bufio.NewReader(h)
err := h.Extension.Dissector.Dissect(b, h.Progress, h.Parent.Origin, h.IsClient, h.TcpID, h.CounterPair, h.SuperTimer, h.Parent.SuperIdentifier, h.Emitter, filteringOptions, h.ReqResMatcher)

27
tap/extensions/amqp/go.mod
View File

@ -4,14 +4,37 @@ go 1.17
require (
github.com/stretchr/testify v1.7.0
github.com/up9inc/mizu/shared v0.0.0
github.com/up9inc/mizu/tap/api v0.0.0
)
require (
github.com/davecgh/go-spew v1.1.0 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/go-logr/logr v1.2.2 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang-jwt/jwt/v4 v4.2.0 // indirect
github.com/google/go-cmp v0.5.7 // indirect
github.com/google/gofuzz v1.2.0 // indirect
github.com/google/gopacket v1.1.19 // indirect
github.com/google/martian v2.1.0+incompatible // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect
golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd // indirect
golang.org/x/text v0.3.7 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
k8s.io/api v0.23.3 // indirect
k8s.io/apimachinery v0.23.3 // indirect
k8s.io/klog/v2 v2.40.1 // indirect
k8s.io/utils v0.0.0-20220127004650-9b3446523e65 // indirect
sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2 // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect
)
replace github.com/up9inc/mizu/tap/api v0.0.0 => ../../api
replace github.com/up9inc/mizu/shared v0.0.0 => ../../../shared

1103
tap/extensions/amqp/go.sum
View File

File diff suppressed because it is too large Load Diff

3
tap/extensions/amqp/main.go
View File

@ -10,6 +10,7 @@ import (
"strconv"
"time"
"github.com/up9inc/mizu/shared"
"github.com/up9inc/mizu/tap/api"
)
@ -39,7 +40,7 @@ func (d dissecting) Ping() {
const amqpRequest string = "amqp_request"
func (d dissecting) Dissect(b *bufio.Reader, progress *api.ReadProgress, capture api.Capture, isClient bool, tcpID *api.TcpID, counterPair *api.CounterPair, superTimer *api.SuperTimer, superIdentifier *api.SuperIdentifier, emitter api.Emitter, options *api.TrafficFilteringOptions, _reqResMatcher api.RequestResponseMatcher) error {
func (d dissecting) Dissect(b *bufio.Reader, progress *api.ReadProgress, capture api.Capture, isClient bool, tcpID *api.TcpID, counterPair *api.CounterPair, superTimer *api.SuperTimer, superIdentifier *api.SuperIdentifier, emitter api.Emitter, options *shared.TrafficFilteringOptions, _reqResMatcher api.RequestResponseMatcher) error {
r := AmqpReader{b}
var remaining int

3
tap/extensions/amqp/main_test.go
View File

@ -15,6 +15,7 @@ import (
"time"
"github.com/stretchr/testify/assert"
"github.com/up9inc/mizu/shared"
"github.com/up9inc/mizu/tap/api"
)
@ -73,7 +74,7 @@ func TestDissect(t *testing.T) {
log.Fatal(err)
}
options := &api.TrafficFilteringOptions{
options := &shared.TrafficFilteringOptions{
IgnoredUserAgents: []string{},
}

21
tap/extensions/http/go.mod
View File

@ -11,10 +11,31 @@ require (
require (
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/go-logr/logr v1.2.2 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang-jwt/jwt/v4 v4.2.0 // indirect
github.com/google/go-cmp v0.5.7 // indirect
github.com/google/gofuzz v1.2.0 // indirect
github.com/google/gopacket v1.1.19 // indirect
github.com/google/martian v2.1.0+incompatible // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/up9inc/mizu/shared v0.0.0 // indirect
golang.org/x/text v0.3.7 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
k8s.io/api v0.23.3 // indirect
k8s.io/apimachinery v0.23.3 // indirect
k8s.io/klog/v2 v2.40.1 // indirect
k8s.io/utils v0.0.0-20220127004650-9b3446523e65 // indirect
sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2 // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect
)
replace github.com/up9inc/mizu/tap/api v0.0.0 => ../../api
replace github.com/up9inc/mizu/shared v0.0.0 => ../../../shared

1089
tap/extensions/http/go.sum
View File

File diff suppressed because it is too large Load Diff

9
tap/extensions/http/handlers.go
View File

@ -9,10 +9,11 @@ import (
"net/http"
"strings"
"github.com/up9inc/mizu/shared"
"github.com/up9inc/mizu/tap/api"
)
func filterAndEmit(item *api.OutputChannelItem, emitter api.Emitter, options *api.TrafficFilteringOptions) {
func filterAndEmit(item *api.OutputChannelItem, emitter api.Emitter, options *shared.TrafficFilteringOptions) {
if IsIgnoredUserAgent(item, options) {
return
}
@ -47,7 +48,7 @@ func replaceForwardedFor(item *api.OutputChannelItem) {
item.ConnectionInfo.ClientPort = ""
}
func handleHTTP2Stream(http2Assembler *Http2Assembler, progress *api.ReadProgress, capture api.Capture, tcpID *api.TcpID, superTimer *api.SuperTimer, emitter api.Emitter, options *api.TrafficFilteringOptions, reqResMatcher *requestResponseMatcher) error {
func handleHTTP2Stream(http2Assembler *Http2Assembler, progress *api.ReadProgress, capture api.Capture, tcpID *api.TcpID, superTimer *api.SuperTimer, emitter api.Emitter, options *shared.TrafficFilteringOptions, reqResMatcher *requestResponseMatcher) error {
streamID, messageHTTP1, isGrpc, err := http2Assembler.readMessage()
if err != nil {
return err
@ -111,7 +112,7 @@ func handleHTTP2Stream(http2Assembler *Http2Assembler, progress *api.ReadProgres
return nil
}
func handleHTTP1ClientStream(b *bufio.Reader, progress *api.ReadProgress, capture api.Capture, tcpID *api.TcpID, counterPair *api.CounterPair, superTimer *api.SuperTimer, emitter api.Emitter, options *api.TrafficFilteringOptions, reqResMatcher *requestResponseMatcher) (switchingProtocolsHTTP2 bool, req *http.Request, err error) {
func handleHTTP1ClientStream(b *bufio.Reader, progress *api.ReadProgress, capture api.Capture, tcpID *api.TcpID, counterPair *api.CounterPair, superTimer *api.SuperTimer, emitter api.Emitter, options *shared.TrafficFilteringOptions, reqResMatcher *requestResponseMatcher) (switchingProtocolsHTTP2 bool, req *http.Request, err error) {
req, err = http.ReadRequest(b)
if err != nil {
return
@ -154,7 +155,7 @@ func handleHTTP1ClientStream(b *bufio.Reader, progress *api.ReadProgress, captur
return
}
func handleHTTP1ServerStream(b *bufio.Reader, progress *api.ReadProgress, capture api.Capture, tcpID *api.TcpID, counterPair *api.CounterPair, superTimer *api.SuperTimer, emitter api.Emitter, options *api.TrafficFilteringOptions, reqResMatcher *requestResponseMatcher) (switchingProtocolsHTTP2 bool, err error) {
func handleHTTP1ServerStream(b *bufio.Reader, progress *api.ReadProgress, capture api.Capture, tcpID *api.TcpID, counterPair *api.CounterPair, superTimer *api.SuperTimer, emitter api.Emitter, options *shared.TrafficFilteringOptions, reqResMatcher *requestResponseMatcher) (switchingProtocolsHTTP2 bool, err error) {
var res *http.Response
res, err = http.ReadResponse(b, nil)
if err != nil {

3
tap/extensions/http/main.go
View File

@ -12,6 +12,7 @@ import (
"strings"
"time"
"github.com/up9inc/mizu/shared"
"github.com/up9inc/mizu/tap/api"
)
@ -86,7 +87,7 @@ func (d dissecting) Ping() {
log.Printf("pong %s", http11protocol.Name)
}
func (d dissecting) Dissect(b *bufio.Reader, progress *api.ReadProgress, capture api.Capture, isClient bool, tcpID *api.TcpID, counterPair *api.CounterPair, superTimer *api.SuperTimer, superIdentifier *api.SuperIdentifier, emitter api.Emitter, options *api.TrafficFilteringOptions, _reqResMatcher api.RequestResponseMatcher) error {
func (d dissecting) Dissect(b *bufio.Reader, progress *api.ReadProgress, capture api.Capture, isClient bool, tcpID *api.TcpID, counterPair *api.CounterPair, superTimer *api.SuperTimer, superIdentifier *api.SuperIdentifier, emitter api.Emitter, options *shared.TrafficFilteringOptions, _reqResMatcher api.RequestResponseMatcher) error {
reqResMatcher := _reqResMatcher.(*requestResponseMatcher)
var err error

3
tap/extensions/http/main_test.go
View File

@ -15,6 +15,7 @@ import (
"time"
"github.com/stretchr/testify/assert"
"github.com/up9inc/mizu/shared"
"github.com/up9inc/mizu/tap/api"
)
@ -75,7 +76,7 @@ func TestDissect(t *testing.T) {
log.Fatal(err)
}
options := &api.TrafficFilteringOptions{
options := &shared.TrafficFilteringOptions{
IgnoredUserAgents: []string{},
}

13
tap/extensions/http/sensitive_data_cleaner.go
View File

@ -12,6 +12,7 @@ import (
"strings"
"github.com/beevik/etree"
"github.com/up9inc/mizu/shared"
"github.com/up9inc/mizu/tap/api"
)
@ -25,7 +26,7 @@ var personallyIdentifiableDataFields = []string{"token", "authorization", "authe
"zip", "zipcode", "address", "country", "firstname", "lastname",
"middlename", "fname", "lname", "birthdate"}
func IsIgnoredUserAgent(item *api.OutputChannelItem, options *api.TrafficFilteringOptions) bool {
func IsIgnoredUserAgent(item *api.OutputChannelItem, options *shared.TrafficFilteringOptions) bool {
if item.Protocol.Name != "http" {
return false
}
@ -49,7 +50,7 @@ func IsIgnoredUserAgent(item *api.OutputChannelItem, options *api.TrafficFilteri
return false
}
func FilterSensitiveData(item *api.OutputChannelItem, options *api.TrafficFilteringOptions) {
func FilterSensitiveData(item *api.OutputChannelItem, options *shared.TrafficFilteringOptions) {
request := item.Pair.Request.Payload.(api.HTTPPayload).Data.(*http.Request)
response := item.Pair.Response.Payload.(api.HTTPPayload).Data.(*http.Response)
@ -60,7 +61,7 @@ func FilterSensitiveData(item *api.OutputChannelItem, options *api.TrafficFilter
filterResponseBody(response, options)
}
func filterRequestBody(request *http.Request, options *api.TrafficFilteringOptions) {
func filterRequestBody(request *http.Request, options *shared.TrafficFilteringOptions) {
contenType := getContentTypeHeaderValue(request.Header)
body, err := ioutil.ReadAll(request.Body)
if err != nil {
@ -74,7 +75,7 @@ func filterRequestBody(request *http.Request, options *api.TrafficFilteringOptio
}
}
func filterResponseBody(response *http.Response, options *api.TrafficFilteringOptions) {
func filterResponseBody(response *http.Response, options *shared.TrafficFilteringOptions) {
contentType := getContentTypeHeaderValue(response.Header)
body, err := ioutil.ReadAll(response.Body)
if err != nil {
@ -130,7 +131,7 @@ func isFieldNameSensitive(fieldName string) bool {
return false
}
func filterHttpBody(bytes []byte, contentType string, options *api.TrafficFilteringOptions) ([]byte, error) {
func filterHttpBody(bytes []byte, contentType string, options *shared.TrafficFilteringOptions) ([]byte, error) {
mimeType := strings.Split(contentType, ";")[0]
switch strings.ToLower(mimeType) {
case "application/json":
@ -151,7 +152,7 @@ func filterHttpBody(bytes []byte, contentType string, options *api.TrafficFilter
return bytes, nil
}
func filterPlainText(bytes []byte, options *api.TrafficFilteringOptions) []byte {
func filterPlainText(bytes []byte, options *shared.TrafficFilteringOptions) []byte {
for _, regex := range options.PlainTextMaskingRegexes {
bytes = regex.ReplaceAll(bytes, []byte(maskedFieldPlaceholderValue))
}

30
tap/extensions/kafka/go.mod
View File

@ -6,19 +6,41 @@ require (
github.com/fatih/camelcase v1.0.0
github.com/ohler55/ojg v1.12.12
github.com/segmentio/kafka-go v0.4.27
github.com/stretchr/testify v1.6.1
github.com/stretchr/testify v1.7.0
github.com/up9inc/mizu/shared v0.0.0
github.com/up9inc/mizu/tap/api v0.0.0
golang.org/x/text v0.3.0
golang.org/x/text v0.3.7
)
require (
github.com/davecgh/go-spew v1.1.0 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/go-logr/logr v1.2.2 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang-jwt/jwt/v4 v4.2.0 // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/google/go-cmp v0.5.7 // indirect
github.com/google/gofuzz v1.2.0 // indirect
github.com/google/gopacket v1.1.19 // indirect
github.com/google/martian v2.1.0+incompatible // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/klauspost/compress v1.14.2 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 // indirect
github.com/pierrec/lz4 v2.6.1+incompatible // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect
golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
k8s.io/api v0.23.3 // indirect
k8s.io/apimachinery v0.23.3 // indirect
k8s.io/klog/v2 v2.40.1 // indirect
k8s.io/utils v0.0.0-20220127004650-9b3446523e65 // indirect
sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2 // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect
)
replace github.com/up9inc/mizu/tap/api v0.0.0 => ../../api
replace github.com/up9inc/mizu/shared v0.0.0 => ../../../shared

1099
tap/extensions/kafka/go.sum
View File

File diff suppressed because it is too large Load Diff

3
tap/extensions/kafka/main.go
View File

@ -8,6 +8,7 @@ import (
"log"
"time"
"github.com/up9inc/mizu/shared"
"github.com/up9inc/mizu/tap/api"
)
@ -35,7 +36,7 @@ func (d dissecting) Ping() {
log.Printf("pong %s", _protocol.Name)
}
func (d dissecting) Dissect(b *bufio.Reader, progress *api.ReadProgress, capture api.Capture, isClient bool, tcpID *api.TcpID, counterPair *api.CounterPair, superTimer *api.SuperTimer, superIdentifier *api.SuperIdentifier, emitter api.Emitter, options *api.TrafficFilteringOptions, _reqResMatcher api.RequestResponseMatcher) error {
func (d dissecting) Dissect(b *bufio.Reader, progress *api.ReadProgress, capture api.Capture, isClient bool, tcpID *api.TcpID, counterPair *api.CounterPair, superTimer *api.SuperTimer, superIdentifier *api.SuperIdentifier, emitter api.Emitter, options *shared.TrafficFilteringOptions, _reqResMatcher api.RequestResponseMatcher) error {
reqResMatcher := _reqResMatcher.(*requestResponseMatcher)
for {
if superIdentifier.Protocol != nil && superIdentifier.Protocol != &_protocol {

3
tap/extensions/kafka/main_test.go
View File

@ -15,6 +15,7 @@ import (
"time"
"github.com/stretchr/testify/assert"
"github.com/up9inc/mizu/shared"
"github.com/up9inc/mizu/tap/api"
)
@ -73,7 +74,7 @@ func TestDissect(t *testing.T) {
log.Fatal(err)
}
options := &api.TrafficFilteringOptions{
options := &shared.TrafficFilteringOptions{
IgnoredUserAgents: []string{},
}

27
tap/extensions/redis/go.mod
View File

@ -4,14 +4,37 @@ go 1.17
require (
github.com/stretchr/testify v1.7.0
github.com/up9inc/mizu/shared v0.0.0
github.com/up9inc/mizu/tap/api v0.0.0
)
require (
github.com/davecgh/go-spew v1.1.0 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/go-logr/logr v1.2.2 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang-jwt/jwt/v4 v4.2.0 // indirect
github.com/google/go-cmp v0.5.7 // indirect
github.com/google/gofuzz v1.2.0 // indirect
github.com/google/gopacket v1.1.19 // indirect
github.com/google/martian v2.1.0+incompatible // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect
golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd // indirect
golang.org/x/text v0.3.7 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
k8s.io/api v0.23.3 // indirect
k8s.io/apimachinery v0.23.3 // indirect
k8s.io/klog/v2 v2.40.1 // indirect
k8s.io/utils v0.0.0-20220127004650-9b3446523e65 // indirect
sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2 // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect
)
replace github.com/up9inc/mizu/tap/api v0.0.0 => ../../api
replace github.com/up9inc/mizu/shared v0.0.0 => ../../../shared

1103
tap/extensions/redis/go.sum
View File

File diff suppressed because it is too large Load Diff

3
tap/extensions/redis/main.go
View File

@ -7,6 +7,7 @@ import (
"log"
"time"
"github.com/up9inc/mizu/shared"
"github.com/up9inc/mizu/tap/api"
)
@ -34,7 +35,7 @@ func (d dissecting) Ping() {
log.Printf("pong %s", protocol.Name)
}
func (d dissecting) Dissect(b *bufio.Reader, progress *api.ReadProgress, capture api.Capture, isClient bool, tcpID *api.TcpID, counterPair *api.CounterPair, superTimer *api.SuperTimer, superIdentifier *api.SuperIdentifier, emitter api.Emitter, options *api.TrafficFilteringOptions, _reqResMatcher api.RequestResponseMatcher) error {
func (d dissecting) Dissect(b *bufio.Reader, progress *api.ReadProgress, capture api.Capture, isClient bool, tcpID *api.TcpID, counterPair *api.CounterPair, superTimer *api.SuperTimer, superIdentifier *api.SuperIdentifier, emitter api.Emitter, options *shared.TrafficFilteringOptions, _reqResMatcher api.RequestResponseMatcher) error {
reqResMatcher := _reqResMatcher.(*requestResponseMatcher)
is := &RedisInputStream{
Reader: b,

3
tap/extensions/redis/main_test.go
View File

@ -16,6 +16,7 @@ import (
"time"
"github.com/stretchr/testify/assert"
"github.com/up9inc/mizu/shared"
"github.com/up9inc/mizu/tap/api"
)
@ -74,7 +75,7 @@ func TestDissect(t *testing.T) {
log.Fatal(err)
}
options := &api.TrafficFilteringOptions{
options := &shared.TrafficFilteringOptions{
IgnoredUserAgents: []string{},
}

17
tap/passive_tapper.go
View File

@ -17,6 +17,7 @@ import (
"strings"
"time"
"github.com/up9inc/mizu/shared"
"github.com/up9inc/mizu/shared/logger"
"github.com/up9inc/mizu/tap/api"
"github.com/up9inc/mizu/tap/api/diagnose"
@ -55,14 +56,14 @@ type TapOpts struct {
HostMode bool
}
var extensions []*api.Extension // global
var filteringOptions *api.TrafficFilteringOptions // global
var tapTargets []v1.Pod // global
var packetSourceManager *source.PacketSourceManager // global
var mainPacketInputChan chan source.TcpPacketInfo // global
var tlsTapperInstance *tlstapper.TlsTapper // global
var extensions []*api.Extension // global
var filteringOptions *shared.TrafficFilteringOptions // global
var tapTargets []v1.Pod // global
var packetSourceManager *source.PacketSourceManager // global
var mainPacketInputChan chan source.TcpPacketInfo // global
var tlsTapperInstance *tlstapper.TlsTapper // global
func StartPassiveTapper(opts *TapOpts, outputItems chan *api.OutputChannelItem, extensionsRef []*api.Extension, options *api.TrafficFilteringOptions) {
func StartPassiveTapper(opts *TapOpts, outputItems chan *api.OutputChannelItem, extensionsRef []*api.Extension, options *shared.TrafficFilteringOptions) {
extensions = extensionsRef
filteringOptions = options
@ -227,7 +228,7 @@ func startPassiveTapper(streamsMap *api.TcpStreamMap, assembler *tcpAssembler) {
logger.Log.Infof("AppStats: %v", diagnose.AppStatsInst)
}
func startTlsTapper(extension *api.Extension, outputItems chan *api.OutputChannelItem, options *api.TrafficFilteringOptions) *tlstapper.TlsTapper {
func startTlsTapper(extension *api.Extension, outputItems chan *api.OutputChannelItem, options *shared.TrafficFilteringOptions) *tlstapper.TlsTapper {
tls := tlstapper.TlsTapper{}
chunksBufferSize := os.Getpagesize() * 100
logBufferSize := os.Getpagesize()

9
tap/tlstapper/tls_poller.go
View File

@ -15,6 +15,7 @@ import (
"github.com/cilium/ebpf/perf"
"github.com/go-errors/errors"
"github.com/up9inc/mizu/shared"
"github.com/up9inc/mizu/shared/logger"
"github.com/up9inc/mizu/tap/api"
)
@ -58,7 +59,7 @@ func (p *tlsPoller) close() error {
return p.chunksReader.Close()
}
func (p *tlsPoller) poll(emitter api.Emitter, options *api.TrafficFilteringOptions) {
func (p *tlsPoller) poll(emitter api.Emitter, options *shared.TrafficFilteringOptions) {
chunks := make(chan *tlsChunk)
go p.pollChunksPerfBuffer(chunks)
@ -115,7 +116,7 @@ func (p *tlsPoller) pollChunksPerfBuffer(chunks chan<- *tlsChunk) {
}
func (p *tlsPoller) handleTlsChunk(chunk *tlsChunk, extension *api.Extension,
emitter api.Emitter, options *api.TrafficFilteringOptions) error {
emitter api.Emitter, options *shared.TrafficFilteringOptions) error {
ip, port, err := chunk.getAddress()
if err != nil {
@ -140,7 +141,7 @@ func (p *tlsPoller) handleTlsChunk(chunk *tlsChunk, extension *api.Extension,
}
func (p *tlsPoller) startNewTlsReader(chunk *tlsChunk, ip net.IP, port uint16, key string, extension *api.Extension,
emitter api.Emitter, options *api.TrafficFilteringOptions) *tlsReader {
emitter api.Emitter, options *shared.TrafficFilteringOptions) *tlsReader {
reader := &tlsReader{
key: key,
@ -163,7 +164,7 @@ func (p *tlsPoller) startNewTlsReader(chunk *tlsChunk, ip net.IP, port uint16, k
}
func dissect(extension *api.Extension, reader *tlsReader, isRequest bool, tcpid *api.TcpID,
tlsEmitter *tlsEmitter, options *api.TrafficFilteringOptions, reqResMatcher api.RequestResponseMatcher) {
tlsEmitter *tlsEmitter, options *shared.TrafficFilteringOptions, reqResMatcher api.RequestResponseMatcher) {
b := bufio.NewReader(reader)
err := extension.Dissector.Dissect(b, reader.progress, api.Ebpf, isRequest, tcpid, &api.CounterPair{},

3
tap/tlstapper/tls_tapper.go
View File

@ -5,6 +5,7 @@ import (
"github.com/cilium/ebpf/rlimit"
"github.com/go-errors/errors"
"github.com/up9inc/mizu/shared"
"github.com/up9inc/mizu/shared/logger"
"github.com/up9inc/mizu/tap/api"
)
@ -50,7 +51,7 @@ func (t *TlsTapper) Init(chunksBufferSize int, logBufferSize int, procfs string,
return t.poller.init(&t.bpfObjects, chunksBufferSize)
}
func (t *TlsTapper) Poll(emitter api.Emitter, options *api.TrafficFilteringOptions) {
func (t *TlsTapper) Poll(emitter api.Emitter, options *shared.TrafficFilteringOptions) {
t.poller.poll(emitter, options)
}