From d7fcf273c016cd41ed15e396abcd89909d214c25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=2E=20Mert=20Y=C4=B1ld=C4=B1ran?= Date: Sun, 17 Apr 2022 09:01:21 -0700 Subject: [PATCH] TRA-4494 Remove all non-functional `OutboundLink` code that was providing `/status/recentTLSLinks` endpoint (#1008) * Remove non-critical TLS detected log that causes `slice bounds out of range` error * Remove all non-functional `OutboundLink` code that was providing `/status/recentTLSLinks` endpoint * Fix more unused code --- agent/go.mod | 1 - agent/go.sum | 2 - agent/pkg/controllers/status_controller.go | 5 +- agent/pkg/models/models.go | 6 --- agent/pkg/providers/status_provider.go | 14 ------ agent/pkg/routes/status_routes.go | 2 - shared/models.go | 3 +- tap/go.mod | 1 - tap/go.sum | 2 - tap/net_utils.go | 15 ------ tap/outboundlinks.go | 39 ---------------- tap/passive_tapper.go | 23 +--------- tap/tcp_reader.go | 44 ++++++------------ tap/tcp_stream_factory.go | 53 ++++++++-------------- 14 files changed, 37 insertions(+), 173 deletions(-) delete mode 100644 tap/outboundlinks.go diff --git a/agent/go.mod b/agent/go.mod index c6810cebd..0cf116374 100644 --- a/agent/go.mod +++ b/agent/go.mod @@ -49,7 +49,6 @@ require ( github.com/PuerkitoBio/purell v1.1.1 // indirect github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect github.com/beevik/etree v1.1.0 // indirect - github.com/bradleyfalzon/tlsx v0.0.0-20170624122154-28fd0e59bac4 // indirect github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5 // indirect github.com/chanced/dynamic v0.0.0-20211210164248-f8fadb1d735b // indirect github.com/cilium/ebpf v0.8.0 // indirect diff --git a/agent/go.sum b/agent/go.sum index 31f58c86b..c88d39dee 100644 --- a/agent/go.sum +++ b/agent/go.sum @@ -108,8 +108,6 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= -github.com/bradleyfalzon/tlsx v0.0.0-20170624122154-28fd0e59bac4 h1:NJOOlc6ZJjix0A1rAU+nxruZtR8KboG1848yqpIUo4M= -github.com/bradleyfalzon/tlsx v0.0.0-20170624122154-28fd0e59bac4/go.mod h1:DQPxZS994Ld1Y8uwnJT+dRL04XPD0cElP/pHH/zEBHM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= diff --git a/agent/pkg/controllers/status_controller.go b/agent/pkg/controllers/status_controller.go index f981e7a29..20330d78d 100644 --- a/agent/pkg/controllers/status_controller.go +++ b/agent/pkg/controllers/status_controller.go @@ -2,6 +2,7 @@ package controllers import ( "net/http" + core "k8s.io/api/core/v1" "github.com/gin-gonic/gin" @@ -93,10 +94,6 @@ func GetGeneralStats(c *gin.Context) { c.JSON(http.StatusOK, providers.GetGeneralStats()) } -func GetRecentTLSLinks(c *gin.Context) { - c.JSON(http.StatusOK, providers.GetAllRecentTLSAddresses()) -} - func GetCurrentResolvingInformation(c *gin.Context) { c.JSON(http.StatusOK, holder.GetResolver().GetMap()) } diff --git a/agent/pkg/models/models.go b/agent/pkg/models/models.go index ef7153101..2fa113357 100644 --- a/agent/pkg/models/models.go +++ b/agent/pkg/models/models.go @@ -9,7 +9,6 @@ import ( basenine "github.com/up9inc/basenine/client/go" "github.com/up9inc/mizu/shared" - "github.com/up9inc/mizu/tap" ) type EntriesRequest struct { @@ -44,11 +43,6 @@ type WebSocketTappedEntryMessage struct { Data *tapApi.OutputChannelItem } -type WebsocketOutboundLinkMessage struct { - *shared.WebSocketMessageMetadata - Data *tap.OutboundLink -} - type AuthStatus struct { Email string `json:"email"` Model string `json:"model"` diff --git a/agent/pkg/providers/status_provider.go b/agent/pkg/providers/status_provider.go index 97b2cd792..4e0a4d17f 100644 --- a/agent/pkg/providers/status_provider.go +++ b/agent/pkg/providers/status_provider.go @@ -9,7 +9,6 @@ import ( "github.com/patrickmn/go-cache" "github.com/up9inc/mizu/agent/pkg/models" "github.com/up9inc/mizu/shared" - "github.com/up9inc/mizu/tap" ) const tlsLinkRetainmentTime = time.Minute * 15 @@ -51,16 +50,3 @@ func GetAuthStatus() (*models.AuthStatus, error) { return authStatus, nil } - -func GetAllRecentTLSAddresses() []string { - recentTLSLinks := make([]string, 0) - - for _, outboundLinkItem := range RecentTLSLinks.Items() { - outboundLink, castOk := outboundLinkItem.Object.(*tap.OutboundLink) - if castOk { - recentTLSLinks = append(recentTLSLinks, outboundLink.DstIP) - } - } - - return recentTLSLinks -} diff --git a/agent/pkg/routes/status_routes.go b/agent/pkg/routes/status_routes.go index 7dacf4235..e83dfa911 100644 --- a/agent/pkg/routes/status_routes.go +++ b/agent/pkg/routes/status_routes.go @@ -21,7 +21,5 @@ func StatusRoutes(ginApp *gin.Engine) { routeGroup.GET("/general", controllers.GetGeneralStats) // get general stats about entries in DB - routeGroup.GET("/recentTLSLinks", controllers.GetRecentTLSLinks) - routeGroup.GET("/resolving", controllers.GetCurrentResolvingInformation) } diff --git a/shared/models.go b/shared/models.go index 25804469e..d2b2e24a4 100644 --- a/shared/models.go +++ b/shared/models.go @@ -20,7 +20,6 @@ const ( WebSocketMessageTypeUpdateStatus WebSocketMessageType = "status" WebSocketMessageTypeUpdateTappedPods WebSocketMessageType = "tappedPods" WebSocketMessageTypeAnalyzeStatus WebSocketMessageType = "analyzeStatus" - WebsocketMessageTypeOutboundLink WebSocketMessageType = "outboundLink" WebSocketMessageTypeToast WebSocketMessageType = "toast" WebSocketMessageTypeQueryMetadata WebSocketMessageType = "queryMetadata" WebSocketMessageTypeStartTime WebSocketMessageType = "startTime" @@ -92,7 +91,7 @@ func (np NodeToPodsMap) Summary() map[string][]string { summary := make(map[string][]string) for node, pods := range np { for _, pod := range pods { - summary[node] = append(summary[node], pod.Namespace + "/" + pod.Name) + summary[node] = append(summary[node], pod.Namespace+"/"+pod.Name) } } diff --git a/tap/go.mod b/tap/go.mod index 2429f2f87..0c99577a0 100644 --- a/tap/go.mod +++ b/tap/go.mod @@ -3,7 +3,6 @@ module github.com/up9inc/mizu/tap go 1.17 require ( - github.com/bradleyfalzon/tlsx v0.0.0-20170624122154-28fd0e59bac4 github.com/cilium/ebpf v0.8.0 github.com/go-errors/errors v1.4.2 github.com/google/gopacket v1.1.19 diff --git a/tap/go.sum b/tap/go.sum index bed20da3d..8963a9190 100644 --- a/tap/go.sum +++ b/tap/go.sum @@ -91,8 +91,6 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= -github.com/bradleyfalzon/tlsx v0.0.0-20170624122154-28fd0e59bac4 h1:NJOOlc6ZJjix0A1rAU+nxruZtR8KboG1848yqpIUo4M= -github.com/bradleyfalzon/tlsx v0.0.0-20170624122154-28fd0e59bac4/go.mod h1:DQPxZS994Ld1Y8uwnJT+dRL04XPD0cElP/pHH/zEBHM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= diff --git a/tap/net_utils.go b/tap/net_utils.go index c60fccbd9..04fe7bce6 100644 --- a/tap/net_utils.go +++ b/tap/net_utils.go @@ -29,21 +29,6 @@ func getLocalhostIPs() ([]string, error) { return myIPs, nil } -//lint:ignore U1000 will be used in the future -func isPrivateIP(ipStr string) bool { - ip := net.ParseIP(ipStr) - if ip.IsLoopback() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() { - return true - } - - for _, block := range privateIPBlocks { - if block.Contains(ip) { - return true - } - } - return false -} - func initPrivateIPBlocks() { for _, cidr := range []string{ "127.0.0.0/8", // IPv4 loopback diff --git a/tap/outboundlinks.go b/tap/outboundlinks.go deleted file mode 100644 index 2150b1f21..000000000 --- a/tap/outboundlinks.go +++ /dev/null @@ -1,39 +0,0 @@ -package tap - -type OutboundLinkProtocol string - -const ( - TLSProtocol OutboundLinkProtocol = "tls" -) - -type OutboundLink struct { - Src string - DstIP string - DstPort int - SuggestedResolvedName string - SuggestedProtocol OutboundLinkProtocol -} - -func NewOutboundLinkWriter() *OutboundLinkWriter { - return &OutboundLinkWriter{ - OutChan: make(chan *OutboundLink), - } -} - -type OutboundLinkWriter struct { - OutChan chan *OutboundLink -} - -func (olw *OutboundLinkWriter) WriteOutboundLink(src string, DstIP string, DstPort int, SuggestedResolvedName string, SuggestedProtocol OutboundLinkProtocol) { - olw.OutChan <- &OutboundLink{ - Src: src, - DstIP: DstIP, - DstPort: DstPort, - SuggestedResolvedName: SuggestedResolvedName, - SuggestedProtocol: SuggestedProtocol, - } -} - -func (olw *OutboundLinkWriter) Stop() { - close(olw.OutChan) -} diff --git a/tap/passive_tapper.go b/tap/passive_tapper.go index 26258f9fe..86c2cb512 100644 --- a/tap/passive_tapper.go +++ b/tap/passive_tapper.go @@ -27,9 +27,6 @@ import ( const cleanPeriod = time.Second * 10 -//lint:ignore U1000 will be used in the future -var remoteOnlyOutboundPorts = []int{80, 443} - var maxcount = flag.Int64("c", -1, "Only grab this many packets, then exit") var decoder = flag.String("decoder", "", "Name of the decoder to use (default: guess from capture)") var statsevery = flag.Int("stats", 60, "Output statistics every N seconds") @@ -58,7 +55,7 @@ var tls = flag.Bool("tls", false, "Enable TLS tapper") var memprofile = flag.String("memprofile", "", "Write memory profile") type TapOpts struct { - HostMode bool + HostMode bool } var extensions []*api.Extension // global @@ -68,24 +65,6 @@ var packetSourceManager *source.PacketSourceManager // global var mainPacketInputChan chan source.TcpPacketInfo // global var tlsTapperInstance *tlstapper.TlsTapper // global -func inArrayInt(arr []int, valueToCheck int) bool { - for _, value := range arr { - if value == valueToCheck { - return true - } - } - return false -} - -func inArrayString(arr []string, valueToCheck string) bool { - for _, value := range arr { - if value == valueToCheck { - return true - } - } - return false -} - func StartPassiveTapper(opts *TapOpts, outputItems chan *api.OutputChannelItem, extensionsRef []*api.Extension, options *api.TrafficFilteringOptions) { extensions = extensionsRef filteringOptions = options diff --git a/tap/tcp_reader.go b/tap/tcp_reader.go index 61b853492..ee0f05f38 100644 --- a/tap/tcp_reader.go +++ b/tap/tcp_reader.go @@ -7,13 +7,10 @@ import ( "sync" "time" - "github.com/bradleyfalzon/tlsx" "github.com/up9inc/mizu/shared/logger" "github.com/up9inc/mizu/tap/api" ) -const checkTLSPacketAmount = 100 - type tcpReaderDataMsg struct { bytes []byte timestamp time.Time @@ -33,22 +30,21 @@ type ConnectionInfo struct { * Implements io.Reader interface (Read) */ type tcpReader struct { - ident string - tcpID *api.TcpID - isClosed bool - isClient bool - isOutgoing bool - msgQueue chan tcpReaderDataMsg // Channel of captured reassembled tcp payload - data []byte - progress *api.ReadProgress - superTimer *api.SuperTimer - parent *tcpStream - packetsSeen uint - outboundLinkWriter *OutboundLinkWriter - extension *api.Extension - emitter api.Emitter - counterPair *api.CounterPair - reqResMatcher api.RequestResponseMatcher + ident string + tcpID *api.TcpID + isClosed bool + isClient bool + isOutgoing bool + msgQueue chan tcpReaderDataMsg // Channel of captured reassembled tcp payload + data []byte + progress *api.ReadProgress + superTimer *api.SuperTimer + parent *tcpStream + packetsSeen uint + extension *api.Extension + emitter api.Emitter + counterPair *api.CounterPair + reqResMatcher api.RequestResponseMatcher sync.Mutex } @@ -64,16 +60,6 @@ func (h *tcpReader) Read(p []byte) (int, error) { if len(h.data) > 0 { h.packetsSeen += 1 } - if h.packetsSeen < checkTLSPacketAmount && len(msg.bytes) > 5 { // packets with less than 5 bytes cause tlsx to panic - clientHello := tlsx.ClientHello{} - err := clientHello.Unmarshall(msg.bytes) - if err == nil { - logger.Log.Debugf("Detected TLS client hello with SNI %s", clientHello.SNI) - // TODO: Throws `panic: runtime error: invalid memory address or nil pointer dereference` error. - // numericPort, _ := strconv.Atoi(h.tcpID.DstPort) - // h.outboundLinkWriter.WriteOutboundLink(h.tcpID.SrcIP, h.tcpID.DstIP, numericPort, clientHello.SNI, TLSProtocol) - } - } } if !ok || len(h.data) == 0 { return 0, io.EOF diff --git a/tap/tcp_stream_factory.go b/tap/tcp_stream_factory.go index 527b6e44d..06ec19f2b 100644 --- a/tap/tcp_stream_factory.go +++ b/tap/tcp_stream_factory.go @@ -20,12 +20,11 @@ import ( * Generates a new tcp stream for each new tcp connection. Closes the stream when the connection closes. */ type tcpStreamFactory struct { - wg sync.WaitGroup - outboundLinkWriter *OutboundLinkWriter - Emitter api.Emitter - streamsMap *tcpStreamMap - ownIps []string - opts *TapOpts + wg sync.WaitGroup + Emitter api.Emitter + streamsMap *tcpStreamMap + ownIps []string + opts *TapOpts } type tcpStreamWrapper struct { @@ -63,9 +62,6 @@ func (factory *tcpStreamFactory) New(net, transport gopacket.Flow, tcp *layers.T srcPort := transport.Src().String() dstPort := transport.Dst().String() - // if factory.shouldNotifyOnOutboundLink(dstIp, dstPort) { - // factory.outboundLinkWriter.WriteOutboundLink(net.Src().String(), dstIp, dstPort, "", "") - // } props := factory.getStreamProps(srcIp, srcPort, dstIp, dstPort) isTapTarget := props.isTapTarget stream := &tcpStream{ @@ -99,14 +95,13 @@ func (factory *tcpStreamFactory) New(net, transport gopacket.Flow, tcp *layers.T SrcPort: srcPort, DstPort: dstPort, }, - parent: stream, - isClient: true, - isOutgoing: props.isOutgoing, - outboundLinkWriter: factory.outboundLinkWriter, - extension: extension, - emitter: factory.Emitter, - counterPair: counterPair, - reqResMatcher: reqResMatcher, + parent: stream, + isClient: true, + isOutgoing: props.isOutgoing, + extension: extension, + emitter: factory.Emitter, + counterPair: counterPair, + reqResMatcher: reqResMatcher, }) stream.servers = append(stream.servers, tcpReader{ msgQueue: make(chan tcpReaderDataMsg), @@ -119,14 +114,13 @@ func (factory *tcpStreamFactory) New(net, transport gopacket.Flow, tcp *layers.T SrcPort: transport.Dst().String(), DstPort: transport.Src().String(), }, - parent: stream, - isClient: false, - isOutgoing: props.isOutgoing, - outboundLinkWriter: factory.outboundLinkWriter, - extension: extension, - emitter: factory.Emitter, - counterPair: counterPair, - reqResMatcher: reqResMatcher, + parent: stream, + isClient: false, + isOutgoing: props.isOutgoing, + extension: extension, + emitter: factory.Emitter, + counterPair: counterPair, + reqResMatcher: reqResMatcher, }) factory.streamsMap.Store(stream.id, &tcpStreamWrapper{ @@ -174,15 +168,6 @@ func (factory *tcpStreamFactory) getStreamProps(srcIP string, srcPort string, ds } } -//lint:ignore U1000 will be used in the future -func (factory *tcpStreamFactory) shouldNotifyOnOutboundLink(dstIP string, dstPort int) bool { - if inArrayInt(remoteOnlyOutboundPorts, dstPort) { - isDirectedHere := inArrayString(factory.ownIps, dstIP) - return !isDirectedHere && !isPrivateIP(dstIP) - } - return true -} - func getPacketOrigin(ac reassembly.AssemblerContext) api.Capture { c, ok := ac.(*context)