Service Map (#623)

* debug builds and gcflags

* update dockerfile for debug

* service map routes and controller

* service map graph structure

* service map interface and new methods

* adding service map edges from mizu entries

* new service map count methods

* implementing the status endpoint

* ServiceMapResponse and ServiceMapEdge models

* service map get endpoint logic

* reset logic and endpoint

* fixed service map get status

* improvements to graph node structure

* front-end implementation and service map buttons

* new render endpoint to render the graph in real time

* spinner sass

* new ServiceMapModal component

* testing react-force-graph-2d lib

* Improvements to service map graph structure, added node id and updated edge source/destination type

* Revert "testing react-force-graph-2d lib"

This reverts commit 1153938386.

* testing react-graph-vis lib

* updated to work with react-graph-vis lib

* removed render endpoint

* go mod tidy

* serviceMap config flag

* using the serviceMap config flag

* passing mizu config to service map as a dependency

* service map tests

* Removed print functions

* finished service map tests

* new service property

* service map controller tests

* moved service map reset button to service map modal
reset closes the modal

* service map modal refresh button and logic

* reset button resets data and refresh

* service map modal close button

* node size/edge size based on the count value
edge color based on protocol

* nodes and edges shadow

* enabled physics to avoid node overlap, changed kafka protocol color to dark green

* showing edges count values and fixed bidirectional edges overlap

* go mod tidy

* removed console.log

* Using the destination node protocol instead of the source node protocol

* Revert "debug builds and gcflags"
Addressed by #624 and #626

This reverts commit 17ecaece3e.

* Revert "update dockerfile for debug"
Addressed by #635

This reverts commit 5dfc15b140.

* using the entire tap Protocol struct instead of only the protocol name

* using the backend protocol background color for node colors

* fixed test, the node list order can change

* re-factoring to get 100% coverage

* using protocol colors just for edges

* re-factored service map to use TCP Entry data. Node key is the entry ip-address instead of the name

* fallback to ip-address when entry name is unresolved

* re-factored front-end

* adjustment to main div style

* added support for multiple protocols for the same edge

* using the item protocol instead of the extension variable

* fixed controller tests

* displaying service name and ip-address on graph nodes

* fixed service map test, we cannot guarantee the slice order

* auth middleware

* created a new pkg for the service map

* re-factoring

* re-factored front-end

* reverting the import order as previous

* Aligning with other UI feature flags handling

* we don't need to get the status anymore, we have window["isServiceMapEnabled"]

* small adjustments

* renamed from .tsx to .ts

* button styles and minor improvements

* moved service map modal from trafficPage to app component

Co-authored-by: Igor Gov <igor.govorov1@gmail.com>
This commit is contained in:
Gustavo Massaneiro
2022-01-19 15:27:12 -03:00
committed by GitHub
parent 7477f867f9
commit d5fd2ff1da
19 changed files with 1384 additions and 88 deletions

29
ui/package-lock.json generated
View File

@@ -14479,6 +14479,25 @@
"resolved": "https://registry.npmjs.org/react-error-overlay/-/react-error-overlay-6.0.9.tgz",
"integrity": "sha512-nQTTcUu+ATDbrSD1BZHr5kgSD4oF8OFjxun8uAaL8RwPBacGBNPf/yAuVVdx17N8XNzRDMrZ9XcKZHCjPW+9ew=="
},
"react-graph-vis": {
"version": "1.0.7",
"resolved": "https://registry.npmjs.org/react-graph-vis/-/react-graph-vis-1.0.7.tgz",
"integrity": "sha512-FI35zlBMKU22JEvG1ukd1DDwW185y4YrDvHm6Bom9EGdA+UNMrZrIV/lyPIRWPcRkzbKaA1w1NvOYcRApD4KdQ==",
"requires": {
"lodash": "^4.17.15",
"prop-types": "^15.5.10",
"uuid": "^2.0.1",
"vis-data": "^7.1.2",
"vis-network": "^9.0.0"
},
"dependencies": {
"uuid": {
"version": "2.0.3",
"resolved": "https://registry.npmjs.org/uuid/-/uuid-2.0.3.tgz",
"integrity": "sha1-Z+LoY3lyFVMN/zGOW/nc6/1Hsho="
}
}
},
"react-is": {
"version": "16.13.1",
"resolved": "https://registry.npmjs.org/react-is/-/react-is-16.13.1.tgz",
@@ -17902,6 +17921,16 @@
"unist-util-stringify-position": "^3.0.0"
}
},
"vis-data": {
"version": "7.1.2",
"resolved": "https://registry.npmjs.org/vis-data/-/vis-data-7.1.2.tgz",
"integrity": "sha512-RPSegFxEcnp3HUEJSzhS2vBdbJ2PSsrYYuhRlpHp2frO/MfRtTYbIkkLZmPkA/Sg3pPfBlR235gcoKbtdm4mbw=="
},
"vis-network": {
"version": "9.1.0",
"resolved": "https://registry.npmjs.org/vis-network/-/vis-network-9.1.0.tgz",
"integrity": "sha512-rx96L144RJWcqOa6afjiFyxZKUerRRbT/YaNMpsusHdwzxrVTO2LlduR45PeJDEztrAf3AU5l2zmiG+1ydUZCw=="
},
"vm-browserify": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/vm-browserify/-/vm-browserify-1.1.2.tgz",

View File

@@ -30,6 +30,7 @@
"react": "^17.0.2",
"react-copy-to-clipboard": "^5.0.3",
"react-dom": "^17.0.2",
"react-graph-vis": "^1.0.7",
"react-lowlight": "^3.0.0",
"react-scripts": "4.0.3",
"react-scrollable-feed-virtualized": "^1.4.9",

View File

@@ -30,6 +30,7 @@
// Injected from server
window.isEnt = __IS_STANDALONE__
window.isOasEnabled = __IS_OAS_ENABLED__
window.isServiceMapEnabled = __IS_SERVICE_MAP_ENABLED__
}
catch (e) {
}

View File

@@ -3,6 +3,7 @@ import './App.sass';
import {TLSWarning} from "./components/TLSWarning/TLSWarning";
import {Header} from "./components/Header/Header";
import {TrafficPage} from "./components/TrafficPage";
import { ServiceMapModal } from './components/ServiceMapModal/ServiceMapModal';
const App = () => {
@@ -10,6 +11,7 @@ const App = () => {
const [showTLSWarning, setShowTLSWarning] = useState(false);
const [userDismissedTLSWarning, setUserDismissedTLSWarning] = useState(false);
const [addressesWithTLS, setAddressesWithTLS] = useState(new Set<string>());
const [openServiceMapModal, setOpenServiceMapModal] = useState(false);
const onTLSDetected = (destAddress: string) => {
addressesWithTLS.add(destAddress);
@@ -22,14 +24,19 @@ const App = () => {
return (
<div className="mizuApp">
<Header analyzeStatus={analyzeStatus}/>
<TrafficPage setAnalyzeStatus={setAnalyzeStatus} onTLSDetected={onTLSDetected}/>
<Header analyzeStatus={analyzeStatus} />
<TrafficPage setAnalyzeStatus={setAnalyzeStatus} onTLSDetected={onTLSDetected} setOpenServiceMapModal={setOpenServiceMapModal} />
<TLSWarning showTLSWarning={showTLSWarning}
setShowTLSWarning={setShowTLSWarning}
addressesWithTLS={addressesWithTLS}
setAddressesWithTLS={setAddressesWithTLS}
userDismissedTLSWarning={userDismissedTLSWarning}
setUserDismissedTLSWarning={setUserDismissedTLSWarning}/>
setShowTLSWarning={setShowTLSWarning}
addressesWithTLS={addressesWithTLS}
setAddressesWithTLS={setAddressesWithTLS}
userDismissedTLSWarning={userDismissedTLSWarning}
setUserDismissedTLSWarning={setUserDismissedTLSWarning} />
{window["isServiceMapEnabled"] && <ServiceMapModal
isOpen={openServiceMapModal}
onOpen={() => setOpenServiceMapModal(true)}
onClose={() => setOpenServiceMapModal(false)}
/>}
</div>
);
}

View File

@@ -11,6 +11,7 @@ import LoadingOverlay from "./components/LoadingOverlay";
import AuthPageBase from './components/AuthPageBase';
import entPageAtom, {Page} from "./recoil/entPage";
import {useRecoilState} from "recoil";
import { ServiceMapModal } from './components/ServiceMapModal/ServiceMapModal';
const api = Api.getInstance();
@@ -22,6 +23,7 @@ const EntApp = () => {
const [addressesWithTLS, setAddressesWithTLS] = useState(new Set<string>());
const [entPage, setEntPage] = useRecoilState(entPageAtom);
const [isFirstLogin, setIsFirstLogin] = useState(false);
const [openServiceMapModal, setOpenServiceMapModal] = useState(false);
const determinePage = useCallback(async () => { // TODO: move to state management
try {
@@ -59,7 +61,7 @@ const EntApp = () => {
switch (entPage) { // TODO: move to state management / proper routing
case Page.Traffic:
pageComponent = <TrafficPage onTLSDetected={onTLSDetected}/>;
pageComponent = <TrafficPage onTLSDetected={onTLSDetected} setOpenServiceMapModal={setOpenServiceMapModal} />;
break;
case Page.Setup:
pageComponent = <AuthPageBase><InstallPage onFirstLogin={() => setIsFirstLogin(true)}/></AuthPageBase>;
@@ -77,14 +79,19 @@ const EntApp = () => {
return (
<div className="mizuApp">
{entPage === Page.Traffic && <EntHeader isFirstLogin={isFirstLogin} setIsFirstLogin={setIsFirstLogin}/>}
{entPage === Page.Traffic && <EntHeader isFirstLogin={isFirstLogin} setIsFirstLogin={setIsFirstLogin} />}
{pageComponent}
{entPage === Page.Traffic && <TLSWarning showTLSWarning={showTLSWarning}
setShowTLSWarning={setShowTLSWarning}
addressesWithTLS={addressesWithTLS}
setAddressesWithTLS={setAddressesWithTLS}
userDismissedTLSWarning={userDismissedTLSWarning}
setUserDismissedTLSWarning={setUserDismissedTLSWarning}/>}
setShowTLSWarning={setShowTLSWarning}
addressesWithTLS={addressesWithTLS}
setAddressesWithTLS={setAddressesWithTLS}
userDismissedTLSWarning={userDismissedTLSWarning}
setUserDismissedTLSWarning={setUserDismissedTLSWarning} />}
{entPage === Page.Traffic && window["isServiceMapEnabled"] && <ServiceMapModal
isOpen={openServiceMapModal}
onOpen={() => setOpenServiceMapModal(true)}
onClose={() => setOpenServiceMapModal(false)}
/>}
</div>
);
}

View File

@@ -0,0 +1,216 @@
import React, { useState, useEffect, useCallback } from "react";
import { Box, Fade, Modal, Backdrop, Button } from "@material-ui/core";
import { toast } from "react-toastify";
import Api from "../../helpers/api";
import spinnerStyle from '../style/Spinner.module.sass';
import spinnerImg from '../assets/spinner.svg';
import Graph from "react-graph-vis";
import debounce from 'lodash/debounce';
import ServiceMapOptions from './ServiceMapOptions'
import { useCommonStyles } from "../../helpers/commonStyle";
interface GraphData {
nodes: Node[];
edges: Edge[];
}
interface Node {
id: number;
value: number;
label: string;
title?: string;
color?: object;
}
interface Edge {
from: number;
to: number;
value: number;
label: string;
title?: string;
color?: object;
}
interface ServiceMapNode {
id: number;
name: string;
entry: Entry;
count: number;
}
interface ServiceMapEdge {
source: ServiceMapNode;
destination: ServiceMapNode;
count: number;
protocol: Protocol;
}
interface ServiceMapGraph {
nodes: ServiceMapNode[];
edges: ServiceMapEdge[];
}
interface Entry {
ip: string;
port: string;
name: string;
}
interface Protocol {
name: string;
abbr: string;
macro: string;
version: string;
backgroundColor: string;
foregroundColor: string;
fontSize: number;
referenceLink: string;
ports: string[];
priority: number;
}
interface ServiceMapModalProps {
isOpen: boolean;
onOpen: () => void;
onClose: () => void;
}
const modalStyle = {
position: 'absolute',
top: '10%',
left: '50%',
transform: 'translate(-50%, 0%)',
width: '80vw',
height: '80vh',
bgcolor: 'background.paper',
borderRadius: '5px',
boxShadow: 24,
p: 4,
color: '#000',
};
const api = Api.getInstance();
export const ServiceMapModal: React.FC<ServiceMapModalProps> = ({ isOpen, onOpen, onClose }) => {
const commonClasses = useCommonStyles();
const [isLoading, setIsLoading] = useState<boolean>(true);
const [graphData, setGraphData] = useState<GraphData>({ nodes: [], edges: [] });
const getServiceMapData = useCallback(async () => {
try {
setIsLoading(true)
const serviceMapData: ServiceMapGraph = await api.serviceMapData()
const newGraphData: GraphData = { nodes: [], edges: [] }
if (serviceMapData.nodes) {
newGraphData.nodes = serviceMapData.nodes.map(node => {
return {
id: node.id,
value: node.count,
label: (node.entry.name === "unresolved") ? node.name : `${node.entry.name} (${node.name})`,
title: "Count: " + node.name,
}
})
}
if (serviceMapData.edges) {
newGraphData.edges = serviceMapData.edges.map(edge => {
return {
from: edge.source.id,
to: edge.destination.id,
value: edge.count,
label: edge.count.toString(),
color: {
color: edge.protocol.backgroundColor,
highlight: edge.protocol.backgroundColor
},
}
})
}
setGraphData(newGraphData)
} catch (ex) {
toast.error("An error occurred while loading Mizu Service Map, see console for mode details");
console.error(ex);
} finally {
setIsLoading(false)
}
// eslint-disable-next-line
}, [isOpen])
useEffect(() => {
getServiceMapData()
}, [getServiceMapData])
const resetServiceMap = debounce(async () => {
try {
const serviceMapResetResponse = await api.serviceMapReset();
if (serviceMapResetResponse["status"] === "enabled") {
refreshServiceMap()
}
} catch (ex) {
toast.error("An error occurred while resetting Mizu Service Map, see console for mode details");
console.error(ex);
}
}, 500);
const refreshServiceMap = debounce(() => {
getServiceMapData();
}, 500);
return (
<Modal
aria-labelledby="transition-modal-title"
aria-describedby="transition-modal-description"
open={isOpen}
onClose={onClose}
closeAfterTransition
BackdropComponent={Backdrop}
BackdropProps={{
timeout: 500,
}}
style={{ overflow: 'auto' }}
>
<Fade in={isOpen}>
<Box sx={modalStyle}>
{isLoading && <div className={spinnerStyle.spinnerContainer}>
<img alt="spinner" src={spinnerImg} style={{ height: 50 }} />
</div>}
{!isLoading && <div style={{ height: "100%", width: "100%" }}>
<Button
variant="contained"
className={commonClasses.button}
style={{ marginRight: 25 }}
onClick={() => onClose()}
>
Close
</Button>
<Button
variant="contained"
className={commonClasses.button}
style={{ marginRight: 25 }}
onClick={resetServiceMap}
>
Reset
</Button>
<Button
variant="contained"
className={commonClasses.button}
onClick={refreshServiceMap}
>
Refresh
</Button>
<Graph
graph={graphData}
options={ServiceMapOptions}
/>
</div>}
</Box>
</Fade>
</Modal>
);
}

View File

@@ -0,0 +1,83 @@
const ServiceMapOptions = {
physics: {
enabled: true,
solver: 'barnesHut',
barnesHut: {
theta: 0.5,
gravitationalConstant: -2000,
centralGravity: 0.3,
springLength: 180,
springConstant: 0.04,
damping: 0.09,
avoidOverlap: 1
},
},
layout: {
hierarchical: false,
randomSeed: 1 // always on node 1
},
nodes: {
shape: 'dot',
chosen: true,
color: {
background: '#27AE60',
border: '#000000',
highlight: {
background: '#27AE60',
border: '#000000',
},
},
font: {
color: '#343434',
size: 14, // px
face: 'arial',
background: 'none',
strokeWidth: 0, // px
strokeColor: '#ffffff',
align: 'center',
multi: false,
},
borderWidth: 1.5,
borderWidthSelected: 2.5,
labelHighlightBold: true,
opacity: 1,
shadow: true,
},
edges: {
chosen: true,
dashes: false,
arrowStrikethrough: false,
arrows: {
to: {
enabled: true,
},
middle: {
enabled: false,
},
from: {
enabled: false,
}
},
smooth: {
enabled: true,
type: 'dynamic',
roundness: 1.0
},
font: {
color: '#343434',
size: 12, // px
face: 'arial',
background: 'none',
strokeWidth: 2, // px
strokeColor: '#ffffff',
align: 'horizontal',
multi: false,
},
labelHighlightBold: true,
selectionWidth: 1,
shadow: true,
},
autoResize: true,
};
export default ServiceMapOptions

View File

@@ -19,6 +19,7 @@ import focusedEntryIdAtom from "../recoil/focusedEntryId";
import websocketConnectionAtom, {WsConnectionStatus} from "../recoil/wsConnection";
import queryAtom from "../recoil/query";
import OasModal from "./OasModal/OasModal";
import {useCommonStyles} from "../helpers/commonStyle"
const useLayoutStyles = makeStyles(() => ({
details: {
@@ -43,11 +44,13 @@ const useLayoutStyles = makeStyles(() => ({
interface TrafficPageProps {
setAnalyzeStatus?: (status: any) => void;
onTLSDetected: (destAddress: string) => void;
setOpenServiceMapModal?: (open: boolean) => void;
}
const api = Api.getInstance();
export const TrafficPage: React.FC<TrafficPageProps> = ({setAnalyzeStatus,onTLSDetected}) => {
export const TrafficPage: React.FC<TrafficPageProps> = ({setAnalyzeStatus,onTLSDetected, setOpenServiceMapModal}) => {
const commonClasses = useCommonStyles();
const classes = useLayoutStyles();
const [tappingStatus, setTappingStatus] = useRecoilState(tappingStatusAtom);
const [entries, setEntries] = useRecoilState(entriesAtom);
@@ -239,79 +242,84 @@ export const TrafficPage: React.FC<TrafficPageProps> = ({setAnalyzeStatus,onTLSD
}
}
return (
<div className="TrafficPage">
<div className="TrafficPageHeader">
<div className="TrafficPageStreamStatus">
<img className="playPauseIcon" style={{visibility: wsConnection === WsConnectionStatus.Connected ? "visible" : "hidden"}} alt="pause"
src={pauseIcon} onClick={toggleConnection}/>
<img className="playPauseIcon" style={{position: "absolute", visibility: wsConnection === WsConnectionStatus.Connected ? "hidden" : "visible"}} alt="play"
src={playIcon} onClick={toggleConnection}/>
<div className="connectionText">
{getConnectionTitle()}
<div className={"indicatorContainer " + getConnectionStatusClass(true)}>
<div className={"indicator " + getConnectionStatusClass(false)}/>
</div>
</div>
</div>
{window["isOasEnabled"] && <div>
<Button
type="submit"
variant="contained"
style={{
margin: "2px 0px 0px 0px",
backgroundColor: variables.blueColor,
fontWeight: 600,
borderRadius: "4px",
color: "#fff",
textTransform: "none",
}}
onClick={handleOpenModal}
>
Show OAS
</Button>
</div>}
const openServiceMapModalDebounce = debounce(() => {
setOpenServiceMapModal(true)
}, 500);
return (
<div className="TrafficPage">
<div className="TrafficPageHeader">
<div className="TrafficPageStreamStatus">
<img className="playPauseIcon" style={{ visibility: wsConnection === WsConnectionStatus.Connected ? "visible" : "hidden" }} alt="pause"
src={pauseIcon} onClick={toggleConnection} />
<img className="playPauseIcon" style={{ position: "absolute", visibility: wsConnection === WsConnectionStatus.Connected ? "hidden" : "visible" }} alt="play"
src={playIcon} onClick={toggleConnection} />
<div className="connectionText">
{getConnectionTitle()}
<div className={"indicatorContainer " + getConnectionStatusClass(true)}>
<div className={"indicator " + getConnectionStatusClass(false)} />
</div>
{window["isOasEnabled"] && <OasModal
openModal={openOasModal}
handleCloseModal={handleCloseModal}
/>}
{<div className="TrafficPage-Container">
<div className="TrafficPage-ListContainer">
<Filters
backgroundColor={queryBackgroundColor}
ws={ws.current}
openWebSocket={openWebSocket}
/>
<div className={styles.container}>
<EntriesList
listEntryREF={listEntry}
onSnapBrokenEvent={onSnapBrokenEvent}
isSnappedToBottom={isSnappedToBottom}
setIsSnappedToBottom={setIsSnappedToBottom}
queriedCurrent={queriedCurrent}
setQueriedCurrent={setQueriedCurrent}
queriedTotal={queriedTotal}
setQueriedTotal={setQueriedTotal}
startTime={startTime}
noMoreDataTop={noMoreDataTop}
setNoMoreDataTop={setNoMoreDataTop}
leftOffTop={leftOffTop}
setLeftOffTop={setLeftOffTop}
ws={ws.current}
openWebSocket={openWebSocket}
leftOffBottom={leftOffBottom}
truncatedTimestamp={truncatedTimestamp}
setTruncatedTimestamp={setTruncatedTimestamp}
scrollableRef={scrollableRef}
/>
</div>
</div>
<div className={classes.details}>
{focusedEntryId && <EntryDetailed/>}
</div>
</div>}
{tappingStatus && !openOasModal && <StatusBar/>}
</div>
</div>
<div style={{ display: 'flex' }}>
{window["isOasEnabled"] && <Button
type="submit"
variant="contained"
className={commonClasses.button}
style={{ marginRight: 25 }}
onClick={handleOpenModal}
>
Show OAS
</Button>}
{window["isServiceMapEnabled"] && <Button
variant="contained"
className={commonClasses.button}
onClick={openServiceMapModalDebounce}
>
Service Map
</Button>}
</div>
</div>
{window["isOasEnabled"] && <OasModal
openModal={openOasModal}
handleCloseModal={handleCloseModal}
/>}
{<div className="TrafficPage-Container">
<div className="TrafficPage-ListContainer">
<Filters
backgroundColor={queryBackgroundColor}
ws={ws.current}
openWebSocket={openWebSocket}
/>
<div className={styles.container}>
<EntriesList
listEntryREF={listEntry}
onSnapBrokenEvent={onSnapBrokenEvent}
isSnappedToBottom={isSnappedToBottom}
setIsSnappedToBottom={setIsSnappedToBottom}
queriedCurrent={queriedCurrent}
setQueriedCurrent={setQueriedCurrent}
queriedTotal={queriedTotal}
setQueriedTotal={setQueriedTotal}
startTime={startTime}
noMoreDataTop={noMoreDataTop}
setNoMoreDataTop={setNoMoreDataTop}
leftOffTop={leftOffTop}
setLeftOffTop={setLeftOffTop}
ws={ws.current}
openWebSocket={openWebSocket}
leftOffBottom={leftOffBottom}
truncatedTimestamp={truncatedTimestamp}
setTruncatedTimestamp={setTruncatedTimestamp}
scrollableRef={scrollableRef}
/>
</div>
</div>
<div className={classes.details}>
{focusedEntryId && <EntryDetailed />}
</div>
</div>}
{tappingStatus && !openOasModal && <StatusBar />}
</div>
);
};

View File

@@ -0,0 +1,7 @@
@import "../../variables.module"
.spinnerContainer
display: flex
justify-content: center
margin-bottom: 10px

View File

@@ -28,6 +28,21 @@ export default class Api {
this.source = null;
}
serviceMapStatus = async () => {
const response = await this.client.get("/servicemap/status");
return response.data;
}
serviceMapData = async () => {
const response = await this.client.get(`/servicemap/get`);
return response.data;
}
serviceMapReset = async () => {
const response = await this.client.get(`/servicemap/reset`);
return response.data;
}
tapStatus = async () => {
const response = await this.client.get("/status/tap");
return response.data;