Service Map (#623)

* debug builds and gcflags

* update dockerfile for debug

* service map routes and controller

* service map graph structure

* service map interface and new methods

* adding service map edges from mizu entries

* new service map count methods

* implementing the status endpoint

* ServiceMapResponse and ServiceMapEdge models

* service map get endpoint logic

* reset logic and endpoint

* fixed service map get status

* improvements to graph node structure

* front-end implementation and service map buttons

* new render endpoint to render the graph in real time

* spinner sass

* new ServiceMapModal component

* testing react-force-graph-2d lib

* Improvements to service map graph structure, added node id and updated edge source/destination type

* Revert "testing react-force-graph-2d lib"

This reverts commit 1153938386.

* testing react-graph-vis lib

* updated to work with react-graph-vis lib

* removed render endpoint

* go mod tidy

* serviceMap config flag

* using the serviceMap config flag

* passing mizu config to service map as a dependency

* service map tests

* Removed print functions

* finished service map tests

* new service property

* service map controller tests

* moved service map reset button to service map modal
reset closes the modal

* service map modal refresh button and logic

* reset button resets data and refresh

* service map modal close button

* node size/edge size based on the count value
edge color based on protocol

* nodes and edges shadow

* enabled physics to avoid node overlap, changed kafka protocol color to dark green

* showing edges count values and fixed bidirectional edges overlap

* go mod tidy

* removed console.log

* Using the destination node protocol instead of the source node protocol

* Revert "debug builds and gcflags"
Addressed by #624 and #626

This reverts commit 17ecaece3e.

* Revert "update dockerfile for debug"
Addressed by #635

This reverts commit 5dfc15b140.

* using the entire tap Protocol struct instead of only the protocol name

* using the backend protocol background color for node colors

* fixed test, the node list order can change

* re-factoring to get 100% coverage

* using protocol colors just for edges

* re-factored service map to use TCP Entry data. Node key is the entry ip-address instead of the name

* fallback to ip-address when entry name is unresolved

* re-factored front-end

* adjustment to main div style

* added support for multiple protocols for the same edge

* using the item protocol instead of the extension variable

* fixed controller tests

* displaying service name and ip-address on graph nodes

* fixed service map test, we cannot guarantee the slice order

* auth middleware

* created a new pkg for the service map

* re-factoring

* re-factored front-end

* reverting the import order as previous

* Aligning with other UI feature flags handling

* we don't need to get the status anymore, we have window["isServiceMapEnabled"]

* small adjustments

* renamed from .tsx to .ts

* button styles and minor improvements

* moved service map modal from trafficPage to app component

Co-authored-by: Igor Gov <igor.govorov1@gmail.com>

ui/src/components/ServiceMapModal/ServiceMapModal.tsx

@@ -0,0 +1,216 @@
+import React, { useState, useEffect, useCallback } from "react";
+import { Box, Fade, Modal, Backdrop, Button } from "@material-ui/core";
+import { toast } from "react-toastify";
+import Api from "../../helpers/api";
+import spinnerStyle from '../style/Spinner.module.sass';
+import spinnerImg from '../assets/spinner.svg';
+import Graph from "react-graph-vis";
+import debounce from 'lodash/debounce';
+import ServiceMapOptions from './ServiceMapOptions'
+import { useCommonStyles } from "../../helpers/commonStyle";
+
+interface GraphData {
+    nodes: Node[];
+    edges: Edge[];
+}
+
+interface Node {
+    id: number;
+    value: number;
+    label: string;
+    title?: string;
+    color?: object;
+}
+
+interface Edge {
+    from: number;
+    to: number;
+    value: number;
+    label: string;
+    title?: string;
+    color?: object;
+}
+
+interface ServiceMapNode {
+    id: number;
+    name: string;
+    entry: Entry;
+    count: number;
+}
+
+interface ServiceMapEdge {
+    source: ServiceMapNode;
+    destination: ServiceMapNode;
+    count: number;
+    protocol: Protocol;
+}
+
+interface ServiceMapGraph {
+    nodes: ServiceMapNode[];
+    edges: ServiceMapEdge[];
+}
+
+interface Entry {
+    ip: string;
+    port: string;
+    name: string;
+}
+
+interface Protocol {
+    name: string;
+    abbr: string;
+    macro: string;
+    version: string;
+    backgroundColor: string;
+    foregroundColor: string;
+    fontSize: number;
+    referenceLink: string;
+    ports: string[];
+    priority: number;
+}
+
+interface ServiceMapModalProps {
+    isOpen: boolean;
+    onOpen: () => void;
+    onClose: () => void;
+}
+
+const modalStyle = {
+    position: 'absolute',
+    top: '10%',
+    left: '50%',
+    transform: 'translate(-50%, 0%)',
+    width: '80vw',
+    height: '80vh',
+    bgcolor: 'background.paper',
+    borderRadius: '5px',
+    boxShadow: 24,
+    p: 4,
+    color: '#000',
+};
+
+const api = Api.getInstance();
+
+export const ServiceMapModal: React.FC<ServiceMapModalProps> = ({ isOpen, onOpen, onClose }) => {
+    const commonClasses = useCommonStyles();
+    const [isLoading, setIsLoading] = useState<boolean>(true);
+    const [graphData, setGraphData] = useState<GraphData>({ nodes: [], edges: [] });
+
+    const getServiceMapData = useCallback(async () => {
+        try {
+            setIsLoading(true)
+
+            const serviceMapData: ServiceMapGraph = await api.serviceMapData()
+            const newGraphData: GraphData = { nodes: [], edges: [] }
+
+            if (serviceMapData.nodes) {
+                newGraphData.nodes = serviceMapData.nodes.map(node => {
+                    return {
+                        id: node.id,
+                        value: node.count,
+                        label: (node.entry.name === "unresolved") ? node.name : `${node.entry.name} (${node.name})`,
+                        title: "Count: " + node.name,
+                    }
+                })
+            }
+
+            if (serviceMapData.edges) {
+                newGraphData.edges = serviceMapData.edges.map(edge => {
+                    return {
+                        from: edge.source.id,
+                        to: edge.destination.id,
+                        value: edge.count,
+                        label: edge.count.toString(),
+                        color: {
+                            color: edge.protocol.backgroundColor,
+                            highlight: edge.protocol.backgroundColor
+                        },
+                    }
+                })
+            }
+
+            setGraphData(newGraphData)
+
+        } catch (ex) {
+            toast.error("An error occurred while loading Mizu Service Map, see console for mode details");
+            console.error(ex);
+        } finally {
+            setIsLoading(false)
+        }
+        // eslint-disable-next-line
+    }, [isOpen])
+
+    useEffect(() => {
+        getServiceMapData()
+    }, [getServiceMapData])
+
+    const resetServiceMap = debounce(async () => {
+        try {
+            const serviceMapResetResponse = await api.serviceMapReset();
+            if (serviceMapResetResponse["status"] === "enabled") {
+                refreshServiceMap()
+            }
+
+        } catch (ex) {
+            toast.error("An error occurred while resetting Mizu Service Map, see console for mode details");
+            console.error(ex);
+        }
+    }, 500);
+
+    const refreshServiceMap = debounce(() => {
+        getServiceMapData();
+    }, 500);
+
+    return (
+        <Modal
+            aria-labelledby="transition-modal-title"
+            aria-describedby="transition-modal-description"
+            open={isOpen}
+            onClose={onClose}
+            closeAfterTransition
+            BackdropComponent={Backdrop}
+            BackdropProps={{
+                timeout: 500,
+            }}
+            style={{ overflow: 'auto' }}
+        >
+            <Fade in={isOpen}>
+                <Box sx={modalStyle}>
+                    {isLoading && <div className={spinnerStyle.spinnerContainer}>
+                        <img alt="spinner" src={spinnerImg} style={{ height: 50 }} />
+                    </div>}
+                    {!isLoading && <div style={{ height: "100%", width: "100%" }}>
+                        <Button
+                            variant="contained"
+                            className={commonClasses.button}
+                            style={{ marginRight: 25 }}
+                            onClick={() => onClose()}
+                        >
+                            Close
+                        </Button>
+                        <Button
+                            variant="contained"
+                            className={commonClasses.button}
+                            style={{ marginRight: 25 }}
+                            onClick={resetServiceMap}
+                        >
+                            Reset
+                        </Button>
+                        <Button
+                            variant="contained"
+                            className={commonClasses.button}
+                            onClick={refreshServiceMap}
+                        >
+                            Refresh
+                        </Button>
+                        <Graph
+                            graph={graphData}
+                            options={ServiceMapOptions}
+                        />
+                    </div>}
+                </Box>
+            </Fade>
+        </Modal>
+    );
+
+}

ui/src/components/ServiceMapModal/ServiceMapOptions.ts

@@ -0,0 +1,83 @@
+const ServiceMapOptions = {
+    physics: {
+        enabled: true,
+        solver: 'barnesHut',
+        barnesHut: {
+            theta: 0.5,
+            gravitationalConstant: -2000,
+            centralGravity: 0.3,
+            springLength: 180,
+            springConstant: 0.04,
+            damping: 0.09,
+            avoidOverlap: 1
+        },
+    },
+    layout: {
+        hierarchical: false,
+        randomSeed: 1 // always on node 1
+    },
+    nodes: {
+        shape: 'dot',
+        chosen: true,
+        color: {
+            background: '#27AE60',
+            border: '#000000',
+            highlight: {
+                background: '#27AE60',
+                border: '#000000',
+            },
+        },
+        font: {
+            color: '#343434',
+            size: 14, // px
+            face: 'arial',
+            background: 'none',
+            strokeWidth: 0, // px
+            strokeColor: '#ffffff',
+            align: 'center',
+            multi: false,
+        },
+        borderWidth: 1.5,
+        borderWidthSelected: 2.5,
+        labelHighlightBold: true,
+        opacity: 1,
+        shadow: true,
+    },
+    edges: {
+        chosen: true,
+        dashes: false,
+        arrowStrikethrough: false,
+        arrows: {
+            to: {
+                enabled: true,
+            },
+            middle: {
+                enabled: false,
+            },
+            from: {
+                enabled: false,
+            }
+        },
+        smooth: {
+            enabled: true,
+            type: 'dynamic',
+            roundness: 1.0
+        },
+        font: {
+            color: '#343434',
+            size: 12, // px
+            face: 'arial',
+            background: 'none',
+            strokeWidth: 2, // px
+            strokeColor: '#ffffff',
+            align: 'horizontal',
+            multi: false,
+        },
+        labelHighlightBold: true,
+        selectionWidth: 1,
+        shadow: true,
+    },
+    autoResize: true,
+};
+
+export default ServiceMapOptions

ui/src/components/TrafficPage.tsx

@@ -19,6 +19,7 @@ import focusedEntryIdAtom from "../recoil/focusedEntryId";
 import websocketConnectionAtom, {WsConnectionStatus} from "../recoil/wsConnection";
 import queryAtom from "../recoil/query";
 import OasModal from "./OasModal/OasModal";
+import {useCommonStyles} from "../helpers/commonStyle"
 
 const useLayoutStyles = makeStyles(() => ({
   details: {
@@ -43,11 +44,13 @@ const useLayoutStyles = makeStyles(() => ({
 interface TrafficPageProps {
   setAnalyzeStatus?: (status: any) => void;
   onTLSDetected: (destAddress: string) => void;
+  setOpenServiceMapModal?: (open: boolean) => void;
 }
 
 const api = Api.getInstance();
 
-export const TrafficPage: React.FC<TrafficPageProps> = ({setAnalyzeStatus,onTLSDetected}) => {
+export const TrafficPage: React.FC<TrafficPageProps> = ({setAnalyzeStatus,onTLSDetected, setOpenServiceMapModal}) => {
+    const commonClasses = useCommonStyles();
     const classes = useLayoutStyles();
     const [tappingStatus, setTappingStatus] = useRecoilState(tappingStatusAtom);
     const [entries, setEntries] = useRecoilState(entriesAtom);
@@ -239,79 +242,84 @@ export const TrafficPage: React.FC<TrafficPageProps> = ({setAnalyzeStatus,onTLSD
         }
     }
 
-    return (
-        <div className="TrafficPage">
-            <div className="TrafficPageHeader">
-              <div className="TrafficPageStreamStatus">
-                  <img className="playPauseIcon" style={{visibility: wsConnection === WsConnectionStatus.Connected ? "visible" : "hidden"}} alt="pause"
-                      src={pauseIcon} onClick={toggleConnection}/>
-                  <img className="playPauseIcon" style={{position: "absolute", visibility: wsConnection === WsConnectionStatus.Connected ? "hidden" : "visible"}} alt="play"
-                      src={playIcon} onClick={toggleConnection}/>
-                  <div className="connectionText">
-                      {getConnectionTitle()}
-                      <div className={"indicatorContainer " + getConnectionStatusClass(true)}>
-                          <div className={"indicator " + getConnectionStatusClass(false)}/>
-                      </div>
-                  </div>
-              </div>
-              {window["isOasEnabled"] && <div>
-                <Button
-                  type="submit"
-                  variant="contained"
-                  style={{
-                    margin: "2px 0px 0px 0px",
-                    backgroundColor: variables.blueColor,
-                    fontWeight: 600,
-                    borderRadius: "4px",
-                    color: "#fff",
-                    textTransform: "none",
-                  }}
-                  onClick={handleOpenModal}
-                >
-                  Show OAS
-                </Button>
-              </div>}
+    const openServiceMapModalDebounce = debounce(() => {
+        setOpenServiceMapModal(true)
+    }, 500);
+
+  return (
+    <div className="TrafficPage">
+      <div className="TrafficPageHeader">
+        <div className="TrafficPageStreamStatus">
+          <img className="playPauseIcon" style={{ visibility: wsConnection === WsConnectionStatus.Connected ? "visible" : "hidden" }} alt="pause"
+            src={pauseIcon} onClick={toggleConnection} />
+          <img className="playPauseIcon" style={{ position: "absolute", visibility: wsConnection === WsConnectionStatus.Connected ? "hidden" : "visible" }} alt="play"
+            src={playIcon} onClick={toggleConnection} />
+          <div className="connectionText">
+            {getConnectionTitle()}
+            <div className={"indicatorContainer " + getConnectionStatusClass(true)}>
+              <div className={"indicator " + getConnectionStatusClass(false)} />
             </div>
-            {window["isOasEnabled"] && <OasModal
-              openModal={openOasModal}
-              handleCloseModal={handleCloseModal}
-            />}
-            {<div className="TrafficPage-Container">
-                <div className="TrafficPage-ListContainer">
-                    <Filters
-                        backgroundColor={queryBackgroundColor}
-                        ws={ws.current}
-                        openWebSocket={openWebSocket}
-                    />
-                    <div className={styles.container}>
-                        <EntriesList
-                            listEntryREF={listEntry}
-                            onSnapBrokenEvent={onSnapBrokenEvent}
-                            isSnappedToBottom={isSnappedToBottom}
-                            setIsSnappedToBottom={setIsSnappedToBottom}
-                            queriedCurrent={queriedCurrent}
-                            setQueriedCurrent={setQueriedCurrent}
-                            queriedTotal={queriedTotal}
-                            setQueriedTotal={setQueriedTotal}
-                            startTime={startTime}
-                            noMoreDataTop={noMoreDataTop}
-                            setNoMoreDataTop={setNoMoreDataTop}
-                            leftOffTop={leftOffTop}
-                            setLeftOffTop={setLeftOffTop}
-                            ws={ws.current}
-                            openWebSocket={openWebSocket}
-                            leftOffBottom={leftOffBottom}
-                            truncatedTimestamp={truncatedTimestamp}
-                            setTruncatedTimestamp={setTruncatedTimestamp}
-                            scrollableRef={scrollableRef}
-                        />
-                    </div>
-                </div>
-                <div className={classes.details}>
-                    {focusedEntryId && <EntryDetailed/>}
-                </div>
-            </div>}
-            {tappingStatus && !openOasModal && <StatusBar/>}
+          </div>
         </div>
+        <div style={{ display: 'flex' }}>
+          {window["isOasEnabled"] && <Button
+            type="submit"
+            variant="contained"
+            className={commonClasses.button}
+            style={{ marginRight: 25 }}
+            onClick={handleOpenModal}
+          >
+            Show OAS
+          </Button>}
+          {window["isServiceMapEnabled"] && <Button
+            variant="contained"
+            className={commonClasses.button}
+            onClick={openServiceMapModalDebounce}
+          >
+            Service Map
+          </Button>}
+        </div>
+      </div>
+      {window["isOasEnabled"] && <OasModal
+        openModal={openOasModal}
+        handleCloseModal={handleCloseModal}
+      />}
+      {<div className="TrafficPage-Container">
+        <div className="TrafficPage-ListContainer">
+          <Filters
+            backgroundColor={queryBackgroundColor}
+            ws={ws.current}
+            openWebSocket={openWebSocket}
+          />
+          <div className={styles.container}>
+            <EntriesList
+              listEntryREF={listEntry}
+              onSnapBrokenEvent={onSnapBrokenEvent}
+              isSnappedToBottom={isSnappedToBottom}
+              setIsSnappedToBottom={setIsSnappedToBottom}
+              queriedCurrent={queriedCurrent}
+              setQueriedCurrent={setQueriedCurrent}
+              queriedTotal={queriedTotal}
+              setQueriedTotal={setQueriedTotal}
+              startTime={startTime}
+              noMoreDataTop={noMoreDataTop}
+              setNoMoreDataTop={setNoMoreDataTop}
+              leftOffTop={leftOffTop}
+              setLeftOffTop={setLeftOffTop}
+              ws={ws.current}
+              openWebSocket={openWebSocket}
+              leftOffBottom={leftOffBottom}
+              truncatedTimestamp={truncatedTimestamp}
+              setTruncatedTimestamp={setTruncatedTimestamp}
+              scrollableRef={scrollableRef}
+            />
+          </div>
+        </div>
+        <div className={classes.details}>
+          {focusedEntryId && <EntryDetailed />}
+        </div>
+      </div>}
+      {tappingStatus && !openOasModal && <StatusBar />}
+    </div>
   );
 };

ui/src/components/style/Spinner.module.sass

@@ -0,0 +1,7 @@
+@import "../../variables.module"
+
+.spinnerContainer
+  display: flex
+  justify-content: center
+  margin-bottom: 10px
+