Optimize UI entry feed performance (#452)

* Optimize the React code for feeding the entries

By building `EntryItem` only once and updating the `entries` state on meta query messages.

* Upgrade `react-scrollable-feed-virtualized` version from `1.4.3` to `1.4.8`

* Fix the `isSelected` state

* Set the query text before deciding the background to prevent lags while typing

* Upgrade Basenine version from `0.2.6` to `0.2.7`

* Set the query background color only if the query is same after the HTTP request and use `useEffect` instead

* Upgrade Basenine version from `0.2.7` to `0.2.8`

* Use `CancelToken` of `axios` instead of trying to check the query state

* Turn `updateQuery` function into a state hook

* Update the macro for `http`

* Do the `source.cancel()` call in `axios.CancelToken`

* Reduce client-side logging

Dockerfile

@@ -42,8 +42,8 @@ RUN go build -ldflags="-s -w \
      -X 'mizuserver/pkg/version.SemVer=${SEM_VER}'" -o mizuagent .
 
 # Download Basenine executable, verify the sha1sum and move it to a directory in $PATH
-ADD https://github.com/up9inc/basenine/releases/download/v0.2.6/basenine_linux_amd64 ./basenine_linux_amd64
+ADD https://github.com/up9inc/basenine/releases/download/v0.2.8/basenine_linux_amd64 ./basenine_linux_amd64
-ADD https://github.com/up9inc/basenine/releases/download/v0.2.6/basenine_linux_amd64.sha256 ./basenine_linux_amd64.sha256
+ADD https://github.com/up9inc/basenine/releases/download/v0.2.8/basenine_linux_amd64.sha256 ./basenine_linux_amd64.sha256
 RUN shasum -a 256 -c basenine_linux_amd64.sha256
 RUN chmod +x ./basenine_linux_amd64
 

agent/go.mod

@@ -16,7 +16,7 @@ require (
 	github.com/op/go-logging v0.0.0-20160315200505-970db520ece7
 	github.com/orcaman/concurrent-map v0.0.0-20210106121528-16402b402231
 	github.com/patrickmn/go-cache v2.1.0+incompatible
-	github.com/up9inc/basenine/client/go v0.0.0-20211109233221-12b405471084
+	github.com/up9inc/basenine/client/go v0.0.0-20211114204315-4d028da5fda5
 	github.com/up9inc/mizu/shared v0.0.0
 	github.com/up9inc/mizu/tap v0.0.0
 	github.com/up9inc/mizu/tap/api v0.0.0

agent/go.sum

@@ -450,8 +450,8 @@ github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo=
 github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=
 github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs=
 github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
-github.com/up9inc/basenine/client/go v0.0.0-20211109233221-12b405471084 h1:gLoP7AyS/c6pYuBQOgALWpzzc5/aSrq98Lr49JRfmfs=
+github.com/up9inc/basenine/client/go v0.0.0-20211114204315-4d028da5fda5 h1:JbLairDLEJpAC8bwmFuOAB+LYpY/oQbzGRSWRpkF7PQ=
-github.com/up9inc/basenine/client/go v0.0.0-20211109233221-12b405471084/go.mod h1:SvJGPoa/6erhUQV7kvHBwM/0x5LyO6XaG2lUaCaKiUI=
+github.com/up9inc/basenine/client/go v0.0.0-20211114204315-4d028da5fda5/go.mod h1:SvJGPoa/6erhUQV7kvHBwM/0x5LyO6XaG2lUaCaKiUI=
 github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw=
 github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f h1:p4VB7kIXpOQvVn1ZaTIVp+3vuYAXFe3OJEvjbUYJLaA=
 github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0=

debug.Dockerfile

@@ -37,8 +37,8 @@ COPY agent .
 RUN go build -gcflags="all=-N -l" -o mizuagent .
 
 # Download Basenine executable, verify the sha1sum and move it to a directory in $PATH
-ADD https://github.com/up9inc/basenine/releases/download/v0.2.6/basenine_linux_amd64 ./basenine_linux_amd64
+ADD https://github.com/up9inc/basenine/releases/download/v0.2.8/basenine_linux_amd64 ./basenine_linux_amd64
-ADD https://github.com/up9inc/basenine/releases/download/v0.2.6/basenine_linux_amd64.sha256 ./basenine_linux_amd64.sha256
+ADD https://github.com/up9inc/basenine/releases/download/v0.2.8/basenine_linux_amd64.sha256 ./basenine_linux_amd64.sha256
 RUN shasum -a 256 -c basenine_linux_amd64.sha256
 RUN chmod +x ./basenine_linux_amd64
 

tap/extensions/http/main.go

@@ -418,7 +418,7 @@ func (d dissecting) Represent(protoIn api.Protocol, request map[string]interface
 
 func (d dissecting) Macros() map[string]string {
 	return map[string]string{
-		`http`:  fmt.Sprintf(`proto.abbr == "%s"`, protocol.Abbreviation),
+		`http`:  fmt.Sprintf(`proto.abbr == "%s" and proto.version == "%s"`, protocol.Abbreviation, protocol.Version),
 		`grpc`:  fmt.Sprintf(`proto.abbr == "%s" and proto.version == "%s"`, protocol.Abbreviation, http2Protocol.Version),
 		`http2`: fmt.Sprintf(`proto.abbr == "%s" and proto.version == "%s"`, protocol.Abbreviation, http2Protocol.Version),
 	}

ui/package-lock.json

@@ -13644,9 +13644,9 @@
       }
     },
     "react-scrollable-feed-virtualized": {
-      "version": "1.4.3",
+      "version": "1.4.8",
-      "resolved": "https://registry.npmjs.org/react-scrollable-feed-virtualized/-/react-scrollable-feed-virtualized-1.4.3.tgz",
+      "resolved": "https://registry.npmjs.org/react-scrollable-feed-virtualized/-/react-scrollable-feed-virtualized-1.4.8.tgz",
-      "integrity": "sha512-M9WgJKr57jCyWKNCksc3oi+xhtO0YbL9d7Ll8Sdc5ZWOIstNvdNbNX0k4Nq6kXUVaHCJ9qE8omdSI/CxT3MLAQ=="
+      "integrity": "sha512-zsSO/9QB+4V6HEk39lxeMEUA6JFSZjfV4stw7RF17+vZdlVhyATsTBCzsj8hZywY4F29cBfH+3/GKrMhwmhAsw=="
     },
     "react-syntax-highlighter": {
       "version": "15.4.3",

ui/package.json

@@ -23,7 +23,7 @@
     "react-copy-to-clipboard": "^5.0.3",
     "react-dom": "^17.0.2",
     "react-scripts": "4.0.3",
-    "react-scrollable-feed-virtualized": "^1.4.3",
+    "react-scrollable-feed-virtualized": "^1.4.8",
     "react-syntax-highlighter": "^15.4.3",
     "react-toastify": "^8.0.3",
     "typescript": "^4.2.4",

ui/src/components/EntriesList.tsx

@@ -1,4 +1,3 @@
-import {EntryItem} from "./EntryListItem/EntryListItem";
 import React, {useRef} from "react";
 import styles from './style/EntriesList.module.sass';
 import ScrollableFeedVirtualized from "react-scrollable-feed-virtualized";
@@ -6,40 +5,32 @@ import down from "./assets/downImg.svg";
 
 interface EntriesListProps {
     entries: any[];
-    setEntries: (entries: any[]) => void;
-    focusedEntryId: string;
-    setFocusedEntryId: (id: string) => void;
     listEntryREF: any;
-    onScrollEvent: (isAtBottom:boolean) => void;
+    onSnapBrokenEvent: () => void;
-    scrollableList: boolean;
+    isSnappedToBottom: boolean;
-    ws: any
+    setIsSnappedToBottom: any;
-    openWebSocket: any;
-    query: string;
-    updateQuery: any;
     queriedCurrent: number;
     queriedTotal: number;
     startTime: number;
 }
 
-export const EntriesList: React.FC<EntriesListProps> = ({entries, setEntries, focusedEntryId, setFocusedEntryId, listEntryREF, onScrollEvent, scrollableList, ws, openWebSocket, query, updateQuery, queriedCurrent, queriedTotal, startTime}) => {
+export const EntriesList: React.FC<EntriesListProps> = ({entries, listEntryREF, onSnapBrokenEvent, isSnappedToBottom, setIsSnappedToBottom, queriedCurrent, queriedTotal, startTime}) => {
 
     const scrollableRef = useRef(null);
 
     return <>
             <div className={styles.list}>
                 <div id="list" ref={listEntryREF} className={styles.list}>
-                    <ScrollableFeedVirtualized ref={scrollableRef} itemHeight={48} marginTop={10} onScroll={(isAtBottom) => onScrollEvent(isAtBottom)}>
+                    <ScrollableFeedVirtualized ref={scrollableRef} itemHeight={48} marginTop={10} onSnapBroken={onSnapBrokenEvent}>
                         {false /* TODO: why there is a need for something here (not necessarily false)? */}
-                        {entries.map(entry => <EntryItem key={entry.id}
+                        {entries}
-                                                        entry={entry}
-                                                        setFocusedEntryId={setFocusedEntryId}
-                                                        isSelected={focusedEntryId === entry.id.toString()}
-                                                        style={{}}
-                                                        updateQuery={updateQuery}/>)}
                     </ScrollableFeedVirtualized>
                     <button type="button"
-                        className={`${styles.btnLive} ${scrollableList ? styles.showButton : styles.hideButton}`}
+                        className={`${styles.btnLive} ${isSnappedToBottom ? styles.hideButton : styles.showButton}`}
-                        onClick={(_) => scrollableRef.current.jumpToBottom()}>
+                        onClick={(_) => {
+                            scrollableRef.current.jumpToBottom();
+                            setIsSnappedToBottom(true);
+                        }}>
                         <img alt="down" src={down} />
                     </button>
                 </div>

ui/src/components/EntryListItem/EntryListItem.tsx

@@ -1,4 +1,4 @@
-import React from "react";
+import React, {useState} from "react";
 import styles from './EntryListItem.module.sass';
 import StatusCode, {getClassification, StatusCodeClassification} from "../UI/StatusCode";
 import Protocol, {ProtocolInterface} from "../UI/Protocol"
@@ -38,12 +38,14 @@ interface Rules {
 interface EntryProps {
     entry: Entry;
     setFocusedEntryId: (id: string) => void;
-    isSelected?: boolean;
     style: object;
     updateQuery: any;
 }
 
-export const EntryItem: React.FC<EntryProps> = ({entry, setFocusedEntryId, isSelected, style, updateQuery}) => {
+export const EntryItem: React.FC<EntryProps> = ({entry, setFocusedEntryId, style, updateQuery}) => {
+
+    const [isSelected, setIsSelected] = useState(false);
+
     const classification = getClassification(entry.statusCode)
     const numberOfRules = entry.rules.numberOfRules
     let ingoingIcon;
@@ -119,7 +121,10 @@ export const EntryItem: React.FC<EntryProps> = ({entry, setFocusedEntryId, isSel
             id={entry.id.toString()}
             className={`${styles.row}
             ${isSelected && !rule && !contractEnabled ? styles.rowSelected : additionalRulesProperties}`}
-            onClick={() => setFocusedEntryId(entry.id.toString())}
+            onClick={() => {
+                setIsSelected(!isSelected);
+                setFocusedEntryId(entry.id.toString());
+            }}
             style={{
                 border: isSelected ? `1px ${entry.protocol.backgroundColor} solid` : "1px transparent solid",
                 position: "absolute",

ui/src/components/TrafficPage.tsx

@@ -1,6 +1,7 @@
 import React, {useEffect, useRef, useState} from "react";
 import {Filters} from "./Filters";
 import {EntriesList} from "./EntriesList";
+import {EntryItem} from "./EntryListItem/EntryListItem";
 import {makeStyles} from "@material-ui/core";
 import "./style/TrafficPage.sass";
 import styles from './style/EntriesList.module.sass';
@@ -50,50 +51,58 @@ export const TrafficPage: React.FC<TrafficPageProps> = ({setAnalyzeStatus, onTLS
     const classes = useLayoutStyles();
 
     const [entries, setEntries] = useState([] as any);
+    const [entriesBuffer, setEntriesBuffer] = useState([] as any);
     const [focusedEntryId, setFocusedEntryId] = useState(null);
     const [selectedEntryData, setSelectedEntryData] = useState(null);
     const [connection, setConnection] = useState(ConnectionStatus.Closed);
 
     const [tappingStatus, setTappingStatus] = useState(null);
 
-    const [disableScrollList, setDisableScrollList] = useState(false);
+    const [isSnappedToBottom, setIsSnappedToBottom] = useState(true);
 
-    const [query, setQueryDefault] = useState("");
+    const [query, setQuery] = useState("");
     const [queryBackgroundColor, setQueryBackgroundColor] = useState("#f5f5f5");
+    const [addition, updateQuery] = useState("");
 
     const [queriedCurrent, setQueriedCurrent] = useState(0);
     const [queriedTotal, setQueriedTotal] = useState(0);
 
     const [startTime, setStartTime] = useState(0);
 
-    const setQuery = async (query) => {
+    useEffect(() => {
-        if (!query) {
+        (async function() {
-            setQueryBackgroundColor("#f5f5f5")
+            if (!query) {
-        } else {
+                setQueryBackgroundColor("#f5f5f5")
-            const data = await api.validateQuery(query);
-            if (data.valid) {
-                setQueryBackgroundColor("#d2fad2")
             } else {
-                setQueryBackgroundColor("#fad6dc")
+                const data = await api.validateQuery(query);
+                if (!data) {
+                    return;
+                }
+                if (data.valid) {
+                    setQueryBackgroundColor("#d2fad2");
+                } else {
+                    setQueryBackgroundColor("#fad6dc");
+                }
             }
-        }
+        })();
-        setQueryDefault(query)
+    }, [query]);
-    }
 
-    const updateQuery = (addition) => {
+    useEffect(() => {
         if (query) {
-            setQuery(`${query} and ${addition}`)
+            setQuery(`${query} and ${addition}`);
         } else {
-            setQuery(addition)
+            setQuery(addition);
         }
-    }
+        // eslint-disable-next-line
+    }, [addition]);
 
     const ws = useRef(null);
 
     const listEntry = useRef(null);
 
     const openWebSocket = (query) => {
-        setEntries([])
+        setEntries([]);
+        setEntriesBuffer([]);
         ws.current = new WebSocket(MizuWebsocketURL);
         ws.current.onopen = () => {
             ws.current.send(query)
@@ -108,15 +117,18 @@ export const TrafficPage: React.FC<TrafficPageProps> = ({setAnalyzeStatus, onTLS
             const message = JSON.parse(e.data);
             switch (message.messageType) {
                 case "entry":
-                    const entry = message.data
+                    const entry = message.data;
-                    if (!focusedEntryId) setFocusedEntryId(entry.id.toString())
+                    if (!focusedEntryId) setFocusedEntryId(entry.id.toString());
-                    let newEntries = [...entries];
+                    setEntriesBuffer([
-                    setEntries([...newEntries, entry])
+                        ...entriesBuffer,
-                    if(listEntry.current) {
+                        <EntryItem
-                        if(isScrollable(listEntry.current.firstChild)) {
+                            key={entry.id}
-                            setDisableScrollList(true)
+                            entry={entry}
-                        }
+                            setFocusedEntryId={setFocusedEntryId}
-                    }
+                            style={{}}
+                            updateQuery={updateQuery}
+                        />
+                    ]);
                     break
                 case "status":
                     setTappingStatus(message.tappingStatus);
@@ -140,8 +152,9 @@ export const TrafficPage: React.FC<TrafficPageProps> = ({setAnalyzeStatus, onTLS
                     });
                     break;
                 case "queryMetadata":
-                    setQueriedCurrent(message.data.current)
+                    setQueriedCurrent(message.data.current);
-                    setQueriedTotal(message.data.total)
+                    setQueriedTotal(message.data.total);
+                    setEntries(entriesBuffer);
                     break;
                 case "startTime":
                     setStartTime(message.data);
@@ -209,21 +222,17 @@ export const TrafficPage: React.FC<TrafficPageProps> = ({setAnalyzeStatus, onTLS
         }
     }
 
-    const onScrollEvent = (isAtBottom) => {
+    const onSnapBrokenEvent = () => {
-        isAtBottom ? setDisableScrollList(false) : setDisableScrollList(true)
+        setIsSnappedToBottom(false)
     }
 
-    const isScrollable = (element) => {
-        return element.scrollHeight > element.clientHeight;
-    };
-
     return (
         <div className="TrafficPage">
             <div className="TrafficPageHeader">
                 <img className="playPauseIcon" style={{visibility: connection === ConnectionStatus.Connected ? "visible" : "hidden"}} alt="pause"
                     src={pauseIcon} onClick={toggleConnection}/>
                 <img className="playPauseIcon" style={{position: "absolute", visibility: connection === ConnectionStatus.Connected ? "hidden" : "visible"}} alt="play"
-                     src={playIcon} onClick={toggleConnection}/>
+                    src={playIcon} onClick={toggleConnection}/>
                 <div className="connectionText">
                     {getConnectionTitle()}
                     <div className={"indicatorContainer " + getConnectionStatusClass(true)}>
@@ -243,16 +252,10 @@ export const TrafficPage: React.FC<TrafficPageProps> = ({setAnalyzeStatus, onTLS
                     <div className={styles.container}>
                         <EntriesList
                             entries={entries}
-                            setEntries={setEntries}
-                            focusedEntryId={focusedEntryId}
-                            setFocusedEntryId={setFocusedEntryId}
                             listEntryREF={listEntry}
-                            onScrollEvent={onScrollEvent}
+                            onSnapBrokenEvent={onSnapBrokenEvent}
-                            scrollableList={disableScrollList}
+                            isSnappedToBottom={isSnappedToBottom}
-                            ws={ws.current}
+                            setIsSnappedToBottom={setIsSnappedToBottom}
-                            openWebSocket={openWebSocket}
-                            query={query}
-                            updateQuery={updateQuery}
                             queriedCurrent={queriedCurrent}
                             queriedTotal={queriedTotal}
                             startTime={startTime}

ui/src/helpers/api.js

@@ -3,6 +3,8 @@ import * as axios from "axios";
 // When working locally cp `cp .env.example .env`
 export const MizuWebsocketURL = process.env.REACT_APP_OVERRIDE_WS_URL ? process.env.REACT_APP_OVERRIDE_WS_URL : `ws://${window.location.host}/ws`;
 
+const CancelToken = axios.CancelToken;
+
 export default class Api {
 
     constructor() {
@@ -17,6 +19,8 @@ export default class Api {
                 Accept: "application/json",
             }
         });
+
+        this.source = null;
     }
 
     tapStatus = async () => {
@@ -45,9 +49,25 @@ export default class Api {
     }
 
     validateQuery = async (query) => {
+        if (this.source) {
+            this.source.cancel();
+        }
+        this.source = CancelToken.source();
+
         const form = new FormData();
         form.append('query', query)
-        const response = await this.client.post(`/query/validate`, form);
+        const response = await this.client.post(`/query/validate`, form, {
+            cancelToken: this.source.token
+        }).catch(function (thrown) {
+            if (!axios.isCancel(thrown)) {
+                console.error('Validate error', thrown.message);
+            }
+        });
+
+        if (!response) {
+            return null;
+        }
+
         return response.data;
     }
 }