From 50c0062db47cc72949b61dc87c8fa36431530a5c Mon Sep 17 00:00:00 2001 From: RamiBerm <54766858+RamiBerm@users.noreply.github.com> Date: Tue, 1 Mar 2022 14:41:23 +0200 Subject: [PATCH 1/7] Support stopping oas/servicemesh/telemetry in flight (#867) * Update oas_generator.go and servicemap.go * Update oas_generator.go * Update esClient.go * Update servicemap.go --- agent/pkg/elastic/esClient.go | 14 +++++++++----- agent/pkg/oas/oas_generator.go | 17 +++++++++++++++-- agent/pkg/servicemap/servicemap.go | 6 ++++++ 3 files changed, 30 insertions(+), 7 deletions(-) diff --git a/agent/pkg/elastic/esClient.go b/agent/pkg/elastic/esClient.go index acedb8579..9ccc30ebb 100644 --- a/agent/pkg/elastic/esClient.go +++ b/agent/pkg/elastic/esClient.go @@ -4,13 +4,14 @@ import ( "bytes" "crypto/tls" "encoding/json" + "net/http" + "sync" + "time" + "github.com/elastic/go-elasticsearch/v7" "github.com/up9inc/mizu/shared" "github.com/up9inc/mizu/shared/logger" "github.com/up9inc/mizu/tap/api" - "net/http" - "sync" - "time" ) type client struct { @@ -31,6 +32,9 @@ func GetInstance() *client { func (client *client) Configure(config shared.ElasticConfig) { if config.Url == "" || config.User == "" || config.Password == "" { + if client.es != nil { + client.es = nil + } logger.Log.Infof("No elastic configuration was supplied, elastic exporter disabled") return } @@ -46,13 +50,13 @@ func (client *client) Configure(config shared.ElasticConfig) { es, err := elasticsearch.NewClient(cfg) if err != nil { - logger.Log.Fatalf("Failed to initialize elastic client %v", err) + logger.Log.Errorf("Failed to initialize elastic client %v", err) } // Have the client instance return a response res, err := es.Info() if err != nil { - logger.Log.Fatalf("Elastic client.Info() ERROR: %v", err) + logger.Log.Errorf("Elastic client.Info() ERROR: %v", err) } else { client.es = es client.index = "mizu_traffic_http_" + time.Now().Format("2006_01_02_15_04") diff --git a/agent/pkg/oas/oas_generator.go b/agent/pkg/oas/oas_generator.go index df7278325..76d9a550b 100644 --- a/agent/pkg/oas/oas_generator.go +++ b/agent/pkg/oas/oas_generator.go @@ -33,10 +33,23 @@ func (g *oasGenerator) Start() { g.entriesChan = make(chan EntryWithSource, 100) // buffer up to 100 entries for OAS processing g.ServiceSpecs = &sync.Map{} g.started = true - go instance.runGeneretor() + go instance.runGenerator() } -func (g *oasGenerator) runGeneretor() { +func (g *oasGenerator) Stop() { + if !g.started { + return + } + g.cancel() + g.Reset() + g.started = false +} + +func (g *oasGenerator) IsStarted() bool { + return g.started +} + +func (g *oasGenerator) runGenerator() { for { select { case <-g.ctx.Done(): diff --git a/agent/pkg/servicemap/servicemap.go b/agent/pkg/servicemap/servicemap.go index 3f7359feb..e27585d3a 100644 --- a/agent/pkg/servicemap/servicemap.go +++ b/agent/pkg/servicemap/servicemap.go @@ -32,6 +32,7 @@ type serviceMap struct { type ServiceMap interface { Enable() + Disable() IsEnabled() bool NewTCPEntry(source *tapApi.TCP, destination *tapApi.TCP, protocol *tapApi.Protocol) GetStatus() ServiceMapStatus @@ -159,6 +160,11 @@ func (s *serviceMap) Enable() { s.enabled = true } +func (s *serviceMap) Disable() { + s.Reset() + s.enabled = false +} + func (s *serviceMap) IsEnabled() bool { return s.enabled } From 94dfa6885897b137137fdfc6501b50fe05d907b7 Mon Sep 17 00:00:00 2001 From: RamiBerm <54766858+RamiBerm@users.noreply.github.com> Date: Wed, 2 Mar 2022 09:48:42 +0200 Subject: [PATCH 2/7] Revert "Support stopping oas/servicemesh/telemetry in flight (#867)" (#868) This reverts commit 50c0062db47cc72949b61dc87c8fa36431530a5c. --- agent/pkg/elastic/esClient.go | 14 +++++--------- agent/pkg/oas/oas_generator.go | 17 ++--------------- agent/pkg/servicemap/servicemap.go | 6 ------ 3 files changed, 7 insertions(+), 30 deletions(-) diff --git a/agent/pkg/elastic/esClient.go b/agent/pkg/elastic/esClient.go index 9ccc30ebb..acedb8579 100644 --- a/agent/pkg/elastic/esClient.go +++ b/agent/pkg/elastic/esClient.go @@ -4,14 +4,13 @@ import ( "bytes" "crypto/tls" "encoding/json" - "net/http" - "sync" - "time" - "github.com/elastic/go-elasticsearch/v7" "github.com/up9inc/mizu/shared" "github.com/up9inc/mizu/shared/logger" "github.com/up9inc/mizu/tap/api" + "net/http" + "sync" + "time" ) type client struct { @@ -32,9 +31,6 @@ func GetInstance() *client { func (client *client) Configure(config shared.ElasticConfig) { if config.Url == "" || config.User == "" || config.Password == "" { - if client.es != nil { - client.es = nil - } logger.Log.Infof("No elastic configuration was supplied, elastic exporter disabled") return } @@ -50,13 +46,13 @@ func (client *client) Configure(config shared.ElasticConfig) { es, err := elasticsearch.NewClient(cfg) if err != nil { - logger.Log.Errorf("Failed to initialize elastic client %v", err) + logger.Log.Fatalf("Failed to initialize elastic client %v", err) } // Have the client instance return a response res, err := es.Info() if err != nil { - logger.Log.Errorf("Elastic client.Info() ERROR: %v", err) + logger.Log.Fatalf("Elastic client.Info() ERROR: %v", err) } else { client.es = es client.index = "mizu_traffic_http_" + time.Now().Format("2006_01_02_15_04") diff --git a/agent/pkg/oas/oas_generator.go b/agent/pkg/oas/oas_generator.go index 76d9a550b..df7278325 100644 --- a/agent/pkg/oas/oas_generator.go +++ b/agent/pkg/oas/oas_generator.go @@ -33,23 +33,10 @@ func (g *oasGenerator) Start() { g.entriesChan = make(chan EntryWithSource, 100) // buffer up to 100 entries for OAS processing g.ServiceSpecs = &sync.Map{} g.started = true - go instance.runGenerator() + go instance.runGeneretor() } -func (g *oasGenerator) Stop() { - if !g.started { - return - } - g.cancel() - g.Reset() - g.started = false -} - -func (g *oasGenerator) IsStarted() bool { - return g.started -} - -func (g *oasGenerator) runGenerator() { +func (g *oasGenerator) runGeneretor() { for { select { case <-g.ctx.Done(): diff --git a/agent/pkg/servicemap/servicemap.go b/agent/pkg/servicemap/servicemap.go index e27585d3a..3f7359feb 100644 --- a/agent/pkg/servicemap/servicemap.go +++ b/agent/pkg/servicemap/servicemap.go @@ -32,7 +32,6 @@ type serviceMap struct { type ServiceMap interface { Enable() - Disable() IsEnabled() bool NewTCPEntry(source *tapApi.TCP, destination *tapApi.TCP, protocol *tapApi.Protocol) GetStatus() ServiceMapStatus @@ -160,11 +159,6 @@ func (s *serviceMap) Enable() { s.enabled = true } -func (s *serviceMap) Disable() { - s.Reset() - s.enabled = false -} - func (s *serviceMap) IsEnabled() bool { return s.enabled } From 9053c58146e677390beaf2965c055e4a1dc24240 Mon Sep 17 00:00:00 2001 From: Liraz Yehezkel Date: Tue, 29 Mar 2022 10:25:25 +0300 Subject: [PATCH 3/7] Remove token from mizu community --- .../Pages/TrafficPage/TrafficPage.tsx | 4 ++-- ui/src/helpers/api.js | 24 ------------------- 2 files changed, 2 insertions(+), 26 deletions(-) diff --git a/ui/src/components/Pages/TrafficPage/TrafficPage.tsx b/ui/src/components/Pages/TrafficPage/TrafficPage.tsx index 3b0419acf..4089ed578 100644 --- a/ui/src/components/Pages/TrafficPage/TrafficPage.tsx +++ b/ui/src/components/Pages/TrafficPage/TrafficPage.tsx @@ -1,6 +1,6 @@ import React, {useEffect, useState} from "react"; import { Button } from "@material-ui/core"; -import Api,{getWebsocketUrl} from "../../../helpers/api"; +import Api, { MizuWebsocketURL } from "../../../helpers/api"; import debounce from 'lodash/debounce'; import {useSetRecoilState, useRecoilState} from "recoil"; import {useCommonStyles} from "../../../helpers/commonStyle" @@ -65,7 +65,7 @@ const trafficViewerApi = {...api} return ( <> - ); diff --git a/ui/src/helpers/api.js b/ui/src/helpers/api.js index 229d8109b..1db29859f 100644 --- a/ui/src/helpers/api.js +++ b/ui/src/helpers/api.js @@ -3,13 +3,10 @@ import * as axios from "axios"; export const MizuWebsocketURL = process.env.REACT_APP_OVERRIDE_WS_URL ? process.env.REACT_APP_OVERRIDE_WS_URL : window.location.protocol === 'https:' ? `wss://${window.location.host}/ws` : `ws://${window.location.host}/ws`; -export const FormValidationErrorType = "formError"; - const CancelToken = axios.CancelToken; const apiURL = process.env.REACT_APP_OVERRIDE_API_URL ? process.env.REACT_APP_OVERRIDE_API_URL : `${window.location.origin}/`; -let token = "" let client = null let source = null @@ -24,8 +21,6 @@ export default class Api { } constructor() { - token = localStorage.getItem("token"); - client = this.getAxiosClient(); source = null; } @@ -125,20 +120,10 @@ export default class Api { return response.data; } - persistToken = (tk) => { - token = tk; - client = this.getAxiosClient(); - localStorage.setItem('token', token); - } - getAxiosClient = () => { const headers = { Accept: "application/json" } - - if (token) { - headers['x-session-token'] = `${token}`; // we use `x-session-token` instead of `Authorization` because the latter is reserved by kubectl proxy, making mizu view not work - } return axios.create({ baseURL: apiURL, timeout: 31000, @@ -146,12 +131,3 @@ export default class Api { }); } } - -export function getWebsocketUrl() { - let websocketUrl = MizuWebsocketURL; - if (token) { - websocketUrl += `/${token}`; - } - - return websocketUrl; -} \ No newline at end of file From 2253737625df0157b9c60304c94ea47076588814 Mon Sep 17 00:00:00 2001 From: Liraz Yehezkel Date: Wed, 30 Mar 2022 13:09:10 +0300 Subject: [PATCH 4/7] Try to avoid overlap only for maps including under 10 services --- .../ServiceMapModal/ServiceMapModal.tsx | 29 ++++++++++++------- .../ServiceMapModal/ServiceMapOptions.ts | 4 +-- 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/ui/src/components/ServiceMapModal/ServiceMapModal.tsx b/ui/src/components/ServiceMapModal/ServiceMapModal.tsx index ca1406fe7..f120ce5b5 100644 --- a/ui/src/components/ServiceMapModal/ServiceMapModal.tsx +++ b/ui/src/components/ServiceMapModal/ServiceMapModal.tsx @@ -100,6 +100,7 @@ export const ServiceMapModal: React.FC = ({ isOpen, onOpen const commonClasses = useCommonStyles(); const [isLoading, setIsLoading] = useState(true); const [graphData, setGraphData] = useState({ nodes: [], edges: [] }); + const [graphOptions, setGraphOptions] = useState(ServiceMapOptions); const getServiceMapData = useCallback(async () => { try { @@ -149,6 +150,14 @@ export const ServiceMapModal: React.FC = ({ isOpen, onOpen // eslint-disable-next-line }, [isOpen]) + useEffect(() => { + if(graphData?.nodes?.length === 0) return; + let options = {...graphOptions}; + options.physics.barnesHut.avoidOverlap = graphData?.nodes?.length > 10 ? 0 : 1; + setGraphOptions(options); + // eslint-disable-next-line + },[graphData?.nodes?.length]) + useEffect(() => { getServiceMapData(); return () => setGraphData({ nodes: [], edges: [] }) @@ -180,7 +189,7 @@ export const ServiceMapModal: React.FC = ({ isOpen, onOpen
- close onClose()} style={{ cursor: "pointer" }}> + close onClose()} style={{ cursor: "pointer" }}/>
    -
  • HTTP
    • -
  • HTTP/2
    • -
  • gRPC
    • -
  • AMQP
    • -
  • KAFKA
    • -
  • REDIS
    • +
  • HTTP
    • +
  • HTTP/2
    • +
  • gRPC
    • +
  • AMQP
    • +
  • KAFKA
    • +
  • REDIS
} @@ -211,4 +220,4 @@ export const ServiceMapModal: React.FC = ({ isOpen, onOpen ); -} \ No newline at end of file +} diff --git a/ui/src/components/ServiceMapModal/ServiceMapOptions.ts b/ui/src/components/ServiceMapModal/ServiceMapOptions.ts index 057ed7101..b525f66dc 100644 --- a/ui/src/components/ServiceMapModal/ServiceMapOptions.ts +++ b/ui/src/components/ServiceMapModal/ServiceMapOptions.ts @@ -148,7 +148,7 @@ const ServiceMapOptions = { springLength: 180, springConstant: 0.04, damping: 0.2, - avoidOverlap: 1 + avoidOverlap: 0 }, }, layout: { @@ -171,4 +171,4 @@ const ServiceMapOptions = { }, }; -export default ServiceMapOptions \ No newline at end of file +export default ServiceMapOptions From 697bfef8612a2bcdc40d52b42c2387081f10862c Mon Sep 17 00:00:00 2001 From: Liraz Yehezkel Date: Wed, 30 Mar 2022 17:03:21 +0300 Subject: [PATCH 5/7] TLS icon position --- .../TrafficViewer/EntryListItem/EntryListItem.module.sass | 4 +++- .../components/TrafficViewer/EntryListItem/EntryListItem.tsx | 4 +--- ui-common/src/components/UI/Protocol.tsx | 2 +- ui-common/src/components/UI/style/StatusCode.module.sass | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/ui-common/src/components/TrafficViewer/EntryListItem/EntryListItem.module.sass b/ui-common/src/components/TrafficViewer/EntryListItem/EntryListItem.module.sass index 334fa0033..b137bb415 100644 --- a/ui-common/src/components/TrafficViewer/EntryListItem/EntryListItem.module.sass +++ b/ui-common/src/components/TrafficViewer/EntryListItem/EntryListItem.module.sass @@ -66,8 +66,10 @@ margin-top: -60px .capture img - height: 20px + height: 14px z-index: 1000 + margin-top: 12px + margin-left: -2px .endpointServiceContainer display: flex diff --git a/ui-common/src/components/TrafficViewer/EntryListItem/EntryListItem.tsx b/ui-common/src/components/TrafficViewer/EntryListItem/EntryListItem.tsx index e55b90fe0..8bc7f0b10 100644 --- a/ui-common/src/components/TrafficViewer/EntryListItem/EntryListItem.tsx +++ b/ui-common/src/components/TrafficViewer/EntryListItem/EntryListItem.tsx @@ -140,8 +140,6 @@ export const EntryItem: React.FC = ({entry, style, headingMode}) => const isStatusCodeEnabled = ((entry.proto.name === "http" && "status" in entry) || entry.status !== 0); - let endpointServiceContainer = "10px"; - if (!isStatusCodeEnabled) endpointServiceContainer = "20px"; return
= ({entry, style, headingMode}) => {isStatusCodeEnabled &&
} -
+
= ({protocol, horizontal}) => { backgroundColor: protocol.backgroundColor, color: protocol.foregroundColor, fontSize: protocol.fontSize, - marginRight: "-20px", + marginRight: "-6px", }} title={protocol.longName} > diff --git a/ui-common/src/components/UI/style/StatusCode.module.sass b/ui-common/src/components/UI/style/StatusCode.module.sass index c9af8a775..4faa4344c 100644 --- a/ui-common/src/components/UI/style/StatusCode.module.sass +++ b/ui-common/src/components/UI/style/StatusCode.module.sass @@ -9,7 +9,7 @@ text-align: center line-height: 22px font-weight: 600 - margin-left: 8px + margin-left: 3px .neutral background: gray From 2ef6afa39583ac6f3bfee4d322a3d201209c66ab Mon Sep 17 00:00:00 2001 From: Nimrod Gilboa Markevich <59927337+nimrod-up9@users.noreply.github.com> Date: Wed, 30 Mar 2022 17:13:56 +0300 Subject: [PATCH 6/7] Hotfix tap issues - restart service mesh tapping when tap targets change, fallback to source namespace (#953) * Read from service mesh network namespaces upon update (#944) #patch * Set the entry namespace to the source namespace if the destination is not resolved (#950) --- agent/pkg/api/main.go | 7 ++++++- tap/passive_tapper.go | 10 +++------- tap/source/packet_source_manager.go | 17 ++++++----------- 3 files changed, 15 insertions(+), 19 deletions(-) diff --git a/agent/pkg/api/main.go b/agent/pkg/api/main.go index d37fbe0da..f4f83b5be 100644 --- a/agent/pkg/api/main.go +++ b/agent/pkg/api/main.go @@ -183,6 +183,7 @@ func resolveIP(connectionInfo *tapApi.ConnectionInfo) (resolvedSource string, re } } else { resolvedSource = resolvedSourceObject.FullAddress + namespace = resolvedSourceObject.Namespace } unresolvedDestination := fmt.Sprintf("%s:%s", connectionInfo.ServerIP, connectionInfo.ServerPort) @@ -194,7 +195,11 @@ func resolveIP(connectionInfo *tapApi.ConnectionInfo) (resolvedSource string, re } } else { resolvedDestination = resolvedDestinationObject.FullAddress - namespace = resolvedDestinationObject.Namespace + // Overwrite namespace (if it was set according to the source) + // Only overwrite if non-empty + if resolvedDestinationObject.Namespace != "" { + namespace = resolvedDestinationObject.Namespace + } } } return resolvedSource, resolvedDestination, namespace diff --git a/tap/passive_tapper.go b/tap/passive_tapper.go index e9ecf5216..57fd2a075 100644 --- a/tap/passive_tapper.go +++ b/tap/passive_tapper.go @@ -112,7 +112,7 @@ func UpdateTapTargets(newTapTargets []v1.Pod) { tapTargets = newTapTargets - packetSourceManager.UpdatePods(tapTargets) + packetSourceManager.UpdatePods(tapTargets, !*nodefrag, mainPacketInputChan) if tlsTapperInstance != nil { if err := tlstapper.UpdateTapTargets(tlsTapperInstance, &tapTargets, *procfs); err != nil { @@ -198,12 +198,8 @@ func initializePacketSources() error { } var err error - if packetSourceManager, err = source.NewPacketSourceManager(*procfs, *fname, *iface, *servicemesh, tapTargets, behaviour); err != nil { - return err - } else { - packetSourceManager.ReadPackets(!*nodefrag, mainPacketInputChan) - return nil - } + packetSourceManager, err = source.NewPacketSourceManager(*procfs, *fname, *iface, *servicemesh, tapTargets, behaviour, !*nodefrag, mainPacketInputChan) + return err } func initializePassiveTapper(opts *TapOpts, outputItems chan *api.OutputChannelItem) (*tcpStreamMap, *tcpAssembler) { diff --git a/tap/source/packet_source_manager.go b/tap/source/packet_source_manager.go index 3fc0ee0b9..bf527ef2e 100644 --- a/tap/source/packet_source_manager.go +++ b/tap/source/packet_source_manager.go @@ -24,7 +24,7 @@ type PacketSourceManager struct { } func NewPacketSourceManager(procfs string, filename string, interfaceName string, - mtls bool, pods []v1.Pod, behaviour TcpPacketSourceBehaviour) (*PacketSourceManager, error) { + mtls bool, pods []v1.Pod, behaviour TcpPacketSourceBehaviour, ipdefrag bool, packets chan<- TcpPacketInfo) (*PacketSourceManager, error) { hostSource, err := newHostPacketSource(filename, interfaceName, behaviour) if err != nil { return nil, err @@ -43,7 +43,7 @@ func NewPacketSourceManager(procfs string, filename string, interfaceName string behaviour: behaviour, } - sourceManager.UpdatePods(pods) + go hostSource.readPackets(ipdefrag, packets) return sourceManager, nil } @@ -64,16 +64,16 @@ func newHostPacketSource(filename string, interfaceName string, return source, nil } -func (m *PacketSourceManager) UpdatePods(pods []v1.Pod) { +func (m *PacketSourceManager) UpdatePods(pods []v1.Pod, ipdefrag bool, packets chan<- TcpPacketInfo) { if m.config.mtls { - m.updateMtlsPods(m.config.procfs, pods, m.config.interfaceName, m.config.behaviour) + m.updateMtlsPods(m.config.procfs, pods, m.config.interfaceName, m.config.behaviour, ipdefrag, packets) } m.setBPFFilter(pods) } func (m *PacketSourceManager) updateMtlsPods(procfs string, pods []v1.Pod, - interfaceName string, behaviour TcpPacketSourceBehaviour) { + interfaceName string, behaviour TcpPacketSourceBehaviour, ipdefrag bool, packets chan<- TcpPacketInfo) { relevantPids := m.getRelevantPids(procfs, pods) logger.Log.Infof("Updating mtls pods (new: %v) (current: %v)", relevantPids, m.sources) @@ -90,6 +90,7 @@ func (m *PacketSourceManager) updateMtlsPods(procfs string, pods []v1.Pod, source, err := newNetnsPacketSource(procfs, pid, interfaceName, behaviour) if err == nil { + go source.readPackets(ipdefrag, packets) m.sources[pid] = source } } @@ -153,12 +154,6 @@ func (m *PacketSourceManager) setBPFFilter(pods []v1.Pod) { } } -func (m *PacketSourceManager) ReadPackets(ipdefrag bool, packets chan<- TcpPacketInfo) { - for _, src := range m.sources { - go src.readPackets(ipdefrag, packets) - } -} - func (m *PacketSourceManager) Close() { for _, src := range m.sources { src.close() From ac94554d2fcf400ec5c1752f65f61dac74557829 Mon Sep 17 00:00:00 2001 From: Liraz Yehezkel Date: Wed, 30 Mar 2022 17:14:56 +0300 Subject: [PATCH 7/7] common versions --- ui-common/package.json | 4 ++-- ui/package.json | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ui-common/package.json b/ui-common/package.json index 4ca4729f7..b0eedc50f 100644 --- a/ui-common/package.json +++ b/ui-common/package.json @@ -1,6 +1,6 @@ { "name": "@up9/mizu-common", - "version": "1.0.144", + "version": "1.0.147", "description": "Made with create-react-library", "author": "", "license": "MIT", @@ -90,4 +90,4 @@ "files": [ "dist" ] -} \ No newline at end of file +} diff --git a/ui/package.json b/ui/package.json index 9a23aedc6..4d997bc0f 100644 --- a/ui/package.json +++ b/ui/package.json @@ -13,7 +13,7 @@ "@types/jest": "^26.0.22", "@types/node": "^12.20.10", "@uiw/react-textarea-code-editor": "^1.4.12", - "@up9/mizu-common": "1.0.144", + "@up9/mizu-common": "1.0.147", "axios": "^0.25.0", "core-js": "^3.20.2", "craco-babel-loader": "^1.0.3", @@ -75,4 +75,4 @@ "last 1 safari version" ] } -} \ No newline at end of file +}