TRA-4017 Bring back getOldEntries method using fetch API and always start streaming from now (#518)

* Bring back `getOldEntries` method using fetch API

* Determine no more data on top based on `leftOff` value

* Remove `entriesBuffer` state

* Always open WebSocket with some `leftOff` value

* Rename `leftOff` state to `leftOffBottom`

* Don't set the `focusedEntryId` through WebSocket if the WebSocket is closed

* Call `setQueriedCurrent` with addition

* Close WebSocket upon reaching to top

* Open WebSocket upon snapping to bottom

* Close the WebSocket on snap broken event instead

* Set queried current value to zero upon filter submit

* Upgrade `react-scrollable-feed-virtualized` version and use `scrollToIndex` function

* Change the footer text format

* Improve no more data top logic

* Fix `closeWebSocket()` call logic in `onSnapBrokenEvent` and handle `data.meta` being `null` in `getOldEntries`

* Fix the issues around fetching old records

* Clean up `EntriesList.module.sass`

* Decrement initial `leftOffTop` value by `2`

* Fix the order of `incomingEntries` in `getOldEntries`

* Request `leftOffTop - 1` from `fetchEntries`

* Limit the front-end total entries fetched through WebSocket count to `10000`

* Lose the UI performance gain that's provided by #452

* Revert "Fix the selected entry behavior by propagating the `focusedEntryId` through WebSocket (before #452) TRA-3983 (#513)"

This reverts commit 873f252544.

* Fix the issues caused by 09371f141f

* Upgrade Basenine version from `0.2.13` to `0.2.14`

* Upgrade Basenine version from `0.2.14` to `0.2.15`

* Fix the condition of "Fetch old records" button visibility

* Upgrade Basenine version from `0.2.15` to `0.2.16` and fix the UI code related to fetching old records

* Make `newEntries` constant

Dockerfile

@@ -42,8 +42,8 @@ RUN go build -ldflags="-s -w \
      -X 'mizuserver/pkg/version.SemVer=${SEM_VER}'" -o mizuagent .
 
 # Download Basenine executable, verify the sha1sum and move it to a directory in $PATH
-ADD https://github.com/up9inc/basenine/releases/download/v0.2.13/basenine_linux_amd64 ./basenine_linux_amd64
+ADD https://github.com/up9inc/basenine/releases/download/v0.2.16/basenine_linux_amd64 ./basenine_linux_amd64
-ADD https://github.com/up9inc/basenine/releases/download/v0.2.13/basenine_linux_amd64.sha256 ./basenine_linux_amd64.sha256
+ADD https://github.com/up9inc/basenine/releases/download/v0.2.16/basenine_linux_amd64.sha256 ./basenine_linux_amd64.sha256
 RUN shasum -a 256 -c basenine_linux_amd64.sha256
 RUN chmod +x ./basenine_linux_amd64
 

agent/go.mod

@@ -16,7 +16,7 @@ require (
 	github.com/op/go-logging v0.0.0-20160315200505-970db520ece7
 	github.com/orcaman/concurrent-map v0.0.0-20210106121528-16402b402231
 	github.com/patrickmn/go-cache v2.1.0+incompatible
-	github.com/up9inc/basenine/client/go v0.0.0-20211130202146-cf837626a065
+	github.com/up9inc/basenine/client/go v0.0.0-20211207165834-2ced7577f9e6
 	github.com/up9inc/mizu/shared v0.0.0
 	github.com/up9inc/mizu/tap v0.0.0
 	github.com/up9inc/mizu/tap/api v0.0.0

agent/go.sum

@@ -450,8 +450,8 @@ github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo=
 github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=
 github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs=
 github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
-github.com/up9inc/basenine/client/go v0.0.0-20211130202146-cf837626a065 h1:kfciLExAWkJMeMoKtnO5G5czqNv5/d0zjupG2nAeBmo=
+github.com/up9inc/basenine/client/go v0.0.0-20211207165834-2ced7577f9e6 h1:8JOkoaZHhUPi4r7vSL/xo83foSz8BHPSabTDpxmtHFU=
-github.com/up9inc/basenine/client/go v0.0.0-20211130202146-cf837626a065/go.mod h1:SvJGPoa/6erhUQV7kvHBwM/0x5LyO6XaG2lUaCaKiUI=
+github.com/up9inc/basenine/client/go v0.0.0-20211207165834-2ced7577f9e6/go.mod h1:SvJGPoa/6erhUQV7kvHBwM/0x5LyO6XaG2lUaCaKiUI=
 github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw=
 github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f h1:p4VB7kIXpOQvVn1ZaTIVp+3vuYAXFe3OJEvjbUYJLaA=
 github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0=

agent/pkg/api/socket_routes.go

@@ -6,7 +6,6 @@ import (
 	"fmt"
 	"mizuserver/pkg/models"
 	"net/http"
-	"strconv"
 	"sync"
 	"time"
 
@@ -95,8 +94,6 @@ func websocketHandler(w http.ResponseWriter, r *http.Request, eventHandlers Even
 	startTimeBytes, _ := models.CreateWebsocketStartTimeMessage(startTime)
 	SendToSocket(socketId, startTimeBytes)
 
-	queryRecieved := false
-
 	for {
 		_, msg, err := ws.ReadMessage()
 		if err != nil {
@@ -104,75 +101,65 @@ func websocketHandler(w http.ResponseWriter, r *http.Request, eventHandlers Even
 			break
 		}
 
-		if !queryRecieved {
+		if !isTapper && !isQuerySet {
-			if !isTapper && !isQuerySet {
+			query := string(msg)
-				queryRecieved = true
+			err = basenine.Validate(shared.BasenineHost, shared.BaseninePort, query)
-				query := string(msg)
-				err = basenine.Validate(shared.BasenineHost, shared.BaseninePort, query)
-				if err != nil {
-					toastBytes, _ := models.CreateWebsocketToastMessage(&models.ToastMessage{
-						Type:      "error",
-						AutoClose: 5000,
-						Text:      fmt.Sprintf("Syntax error: %s", err.Error()),
-					})
-					SendToSocket(socketId, toastBytes)
-					break
-				}
-
-				isQuerySet = true
-
-				handleDataChannel := func(c *basenine.Connection, data chan []byte) {
-					for {
-						bytes := <-data
-
-						if string(bytes) == basenine.CloseChannel {
-							return
-						}
-
-						var dataMap map[string]interface{}
-						err = json.Unmarshal(bytes, &dataMap)
-
-						base := dataMap["base"].(map[string]interface{})
-						base["id"] = uint(dataMap["id"].(float64))
-
-						baseEntryBytes, _ := models.CreateBaseEntryWebSocketMessage(base)
-						SendToSocket(socketId, baseEntryBytes)
-					}
-				}
-
-				handleMetaChannel := func(c *basenine.Connection, meta chan []byte) {
-					for {
-						bytes := <-meta
-
-						if string(bytes) == basenine.CloseChannel {
-							return
-						}
-
-						var metadata *basenine.Metadata
-						err = json.Unmarshal(bytes, &metadata)
-						if err != nil {
-							logger.Log.Debugf("Error recieving metadata: %v", err.Error())
-						}
-
-						metadataBytes, _ := models.CreateWebsocketQueryMetadataMessage(metadata)
-						SendToSocket(socketId, metadataBytes)
-					}
-				}
-
-				go handleDataChannel(connection, data)
-				go handleMetaChannel(connection, meta)
-
-				connection.Query(query, data, meta)
-			} else {
-				eventHandlers.WebSocketMessage(socketId, msg)
-			}
-		} else {
-			id, err := strconv.Atoi(string(msg))
 			if err != nil {
-				continue
+				toastBytes, _ := models.CreateWebsocketToastMessage(&models.ToastMessage{
+					Type:      "error",
+					AutoClose: 5000,
+					Text:      fmt.Sprintf("Syntax error: %s", err.Error()),
+				})
+				SendToSocket(socketId, toastBytes)
+				break
 			}
-			focusEntryBytes, _ := models.CreateWebsocketFocusEntry(id)
+
-			SendToSocket(socketId, focusEntryBytes)
+			isQuerySet = true
+
+			handleDataChannel := func(c *basenine.Connection, data chan []byte) {
+				for {
+					bytes := <-data
+
+					if string(bytes) == basenine.CloseChannel {
+						return
+					}
+
+					var dataMap map[string]interface{}
+					err = json.Unmarshal(bytes, &dataMap)
+
+					base := dataMap["base"].(map[string]interface{})
+					base["id"] = uint(dataMap["id"].(float64))
+
+					baseEntryBytes, _ := models.CreateBaseEntryWebSocketMessage(base)
+					SendToSocket(socketId, baseEntryBytes)
+				}
+			}
+
+			handleMetaChannel := func(c *basenine.Connection, meta chan []byte) {
+				for {
+					bytes := <-meta
+
+					if string(bytes) == basenine.CloseChannel {
+						return
+					}
+
+					var metadata *basenine.Metadata
+					err = json.Unmarshal(bytes, &metadata)
+					if err != nil {
+						logger.Log.Debugf("Error recieving metadata: %v", err.Error())
+					}
+
+					metadataBytes, _ := models.CreateWebsocketQueryMetadataMessage(metadata)
+					SendToSocket(socketId, metadataBytes)
+				}
+			}
+
+			go handleDataChannel(connection, data)
+			go handleMetaChannel(connection, meta)
+
+			connection.Query(query, data, meta)
+		} else {
+			eventHandlers.WebSocketMessage(socketId, msg)
 		}
 	}
 }

agent/pkg/models/models.go

@@ -70,11 +70,6 @@ type WebSocketStartTimeMessage struct {
 	Data int64 `json:"data"`
 }
 
-type WebSocketFocusEntryMessage struct {
-	*shared.WebSocketMessageMetadata
-	Id int `json:"id"`
-}
-
 func CreateBaseEntryWebSocketMessage(base map[string]interface{}) ([]byte, error) {
 	message := &WebSocketEntryMessage{
 		WebSocketMessageMetadata: &shared.WebSocketMessageMetadata{
@@ -135,16 +130,6 @@ func CreateWebsocketStartTimeMessage(base int64) ([]byte, error) {
 	return json.Marshal(message)
 }
 
-func CreateWebsocketFocusEntry(id int) ([]byte, error) {
-	message := &WebSocketFocusEntryMessage{
-		WebSocketMessageMetadata: &shared.WebSocketMessageMetadata{
-			MessageType: shared.WebSocketMessageFocusEntry,
-		},
-		Id: id,
-	}
-	return json.Marshal(message)
-}
-
 // ExtendedHAR is the top level object of a HAR log.
 type ExtendedHAR struct {
 	Log *ExtendedLog `json:"log"`

debug.Dockerfile

@@ -37,8 +37,8 @@ COPY agent .
 RUN go build -gcflags="all=-N -l" -o mizuagent .
 
 # Download Basenine executable, verify the sha1sum and move it to a directory in $PATH
-ADD https://github.com/up9inc/basenine/releases/download/v0.2.13/basenine_linux_amd64 ./basenine_linux_amd64
+ADD https://github.com/up9inc/basenine/releases/download/v0.2.16/basenine_linux_amd64 ./basenine_linux_amd64
-ADD https://github.com/up9inc/basenine/releases/download/v0.2.13/basenine_linux_amd64.sha256 ./basenine_linux_amd64.sha256
+ADD https://github.com/up9inc/basenine/releases/download/v0.2.16/basenine_linux_amd64.sha256 ./basenine_linux_amd64.sha256
 RUN shasum -a 256 -c basenine_linux_amd64.sha256
 RUN chmod +x ./basenine_linux_amd64
 

shared/models.go

@@ -1,12 +1,11 @@
 package shared
 
 import (
-	"io/ioutil"
-	"strings"
-
 	"github.com/op/go-logging"
 	"github.com/up9inc/mizu/shared/logger"
 	"github.com/up9inc/mizu/tap/api"
+	"io/ioutil"
+	"strings"
 
 	"gopkg.in/yaml.v3"
 )
@@ -22,7 +21,6 @@ const (
 	WebSocketMessageTypeToast         WebSocketMessageType = "toast"
 	WebSocketMessageTypeQueryMetadata WebSocketMessageType = "queryMetadata"
 	WebSocketMessageTypeStartTime     WebSocketMessageType = "startTime"
-	WebSocketMessageFocusEntry        WebSocketMessageType = "focusEntry"
 )
 
 type Resources struct {

ui/package-lock.json

@@ -13644,9 +13644,9 @@
       }
     },
     "react-scrollable-feed-virtualized": {
-      "version": "1.4.8",
+      "version": "1.4.9",
-      "resolved": "https://registry.npmjs.org/react-scrollable-feed-virtualized/-/react-scrollable-feed-virtualized-1.4.8.tgz",
+      "resolved": "https://registry.npmjs.org/react-scrollable-feed-virtualized/-/react-scrollable-feed-virtualized-1.4.9.tgz",
-      "integrity": "sha512-zsSO/9QB+4V6HEk39lxeMEUA6JFSZjfV4stw7RF17+vZdlVhyATsTBCzsj8hZywY4F29cBfH+3/GKrMhwmhAsw=="
+      "integrity": "sha512-YkFkPjdIXDUsaCNYhZ+Blpp3LF+CsJWscwn/0fGSjF5QBKCtPURO9AEUA362Qnjr4S8LF2IjSAOCCFedIEnVNw=="
     },
     "react-syntax-highlighter": {
       "version": "15.4.3",

ui/package.json

@@ -23,7 +23,7 @@
     "react-copy-to-clipboard": "^5.0.3",
     "react-dom": "^17.0.2",
     "react-scripts": "4.0.3",
-    "react-scrollable-feed-virtualized": "^1.4.8",
+    "react-scrollable-feed-virtualized": "^1.4.9",
     "react-syntax-highlighter": "^15.4.3",
     "react-toastify": "^8.0.3",
     "typescript": "^4.2.4",

ui/src/components/EntriesList.tsx

@@ -1,33 +1,132 @@
-import React, {useRef} from "react";
+import React, {useCallback, useEffect, useMemo, useRef, useState} from "react";
 import styles from './style/EntriesList.module.sass';
 import ScrollableFeedVirtualized from "react-scrollable-feed-virtualized";
+import {EntryItem} from "./EntryListItem/EntryListItem";
 import down from "./assets/downImg.svg";
+import spinner from './assets/spinner.svg';
+import Api from "../helpers/api";
 
 interface EntriesListProps {
     entries: any[];
+    setEntries: any;
+    query: string;
     listEntryREF: any;
     onSnapBrokenEvent: () => void;
     isSnappedToBottom: boolean;
     setIsSnappedToBottom: any;
     queriedCurrent: number;
+    setQueriedCurrent: any;
     queriedTotal: number;
     startTime: number;
+    noMoreDataTop: boolean;
+    setNoMoreDataTop: (flag: boolean) => void;
+    focusedEntryId: string;
+    setFocusedEntryId: (id: string) => void;
+    updateQuery: any;
+    leftOffTop: number;
+    setLeftOffTop: (leftOffTop: number) => void;
+    reconnectWebSocket: any;
+    isWebSocketConnectionClosed: boolean;
+    closeWebSocket: any;
 }
 
-export const EntriesList: React.FC<EntriesListProps> = ({entries, listEntryREF, onSnapBrokenEvent, isSnappedToBottom, setIsSnappedToBottom, queriedCurrent, queriedTotal, startTime}) => {
+const api = new Api();
 
+export const EntriesList: React.FC<EntriesListProps> = ({entries, setEntries, query, listEntryREF, onSnapBrokenEvent, isSnappedToBottom, setIsSnappedToBottom, queriedCurrent, setQueriedCurrent, queriedTotal, startTime, noMoreDataTop, setNoMoreDataTop, focusedEntryId, setFocusedEntryId, updateQuery, leftOffTop, setLeftOffTop, reconnectWebSocket, isWebSocketConnectionClosed, closeWebSocket}) => {
+    const [loadMoreTop, setLoadMoreTop] = useState(false);
+    const [isLoadingTop, setIsLoadingTop] = useState(false);
     const scrollableRef = useRef(null);
 
+    useEffect(() => {
+        const list = document.getElementById('list').firstElementChild;
+        list.addEventListener('scroll', (e) => {
+            const el: any = e.target;
+            if(el.scrollTop === 0) {
+                setLoadMoreTop(true);
+            } else {
+                setNoMoreDataTop(false);
+                setLoadMoreTop(false);
+            }
+        });
+    }, [setLoadMoreTop, setNoMoreDataTop]);
+
+    const memoizedEntries = useMemo(() => {
+        return entries;
+    },[entries]);
+
+    const getOldEntries = useCallback(async () => {
+        setLoadMoreTop(false);
+        if (leftOffTop === null || leftOffTop <= 0) {
+            return;
+        }
+        setIsLoadingTop(true);
+        const data = await api.fetchEntries(leftOffTop, -1, query, 100, 3000);
+        if (!data || !data.meta) {
+            setNoMoreDataTop(true);
+            setIsLoadingTop(false);
+            return;
+        }
+        setLeftOffTop(data.meta.leftOff);
+
+        let scrollTo: boolean;
+        if (data.meta.leftOff === 0) {
+            setNoMoreDataTop(true);
+            scrollTo = false;
+        } else {
+            scrollTo = true;
+        }
+        setIsLoadingTop(false);
+
+        const newEntries = [...data.data.reverse(), ...entries];
+        setEntries(newEntries);
+
+        setQueriedCurrent(queriedCurrent + data.meta.current);
+
+        if (scrollTo) {
+            scrollableRef.current.scrollToIndex(data.data.length - 1);
+        }
+    },[setLoadMoreTop, setIsLoadingTop, entries, setEntries, query, setNoMoreDataTop, leftOffTop, setLeftOffTop, queriedCurrent, setQueriedCurrent]);
+
+    useEffect(() => {
+        if(!isWebSocketConnectionClosed || !loadMoreTop || noMoreDataTop) return;
+        getOldEntries();
+    }, [loadMoreTop, noMoreDataTop, getOldEntries, isWebSocketConnectionClosed]);
+
+    const scrollbarVisible = scrollableRef.current?.childWrapperRef.current.clientHeight > scrollableRef.current?.wrapperRef.current.clientHeight;
+
     return <>
             <div className={styles.list}>
                 <div id="list" ref={listEntryREF} className={styles.list}>
+                    {isLoadingTop && <div className={styles.spinnerContainer}>
+                        <img alt="spinner" src={spinner} style={{height: 25}}/>
+                    </div>}
+                    {noMoreDataTop && <div id="noMoreDataTop" className={styles.noMoreDataAvailable}>No more data available</div>}
                     <ScrollableFeedVirtualized ref={scrollableRef} itemHeight={48} marginTop={10} onSnapBroken={onSnapBrokenEvent}>
-                        {false /* TODO: why there is a need for something here (not necessarily false)? */}
+                        {false /* It's because the first child is ignored by ScrollableFeedVirtualized */}
-                        {entries}
+                        {memoizedEntries.map(entry => <EntryItem
+                            key={`entry-${entry.id}`}
+                            entry={entry}
+                            focusedEntryId={focusedEntryId}
+                            setFocusedEntryId={setFocusedEntryId}
+                            style={{}}
+                            updateQuery={updateQuery}
+                            headingMode={false}
+                        />)}
                     </ScrollableFeedVirtualized>
                     <button type="button"
-                        className={`${styles.btnLive} ${isSnappedToBottom ? styles.hideButton : styles.showButton}`}
+                        title="Fetch old records"
+                        className={`${styles.btnOld} ${!scrollbarVisible && leftOffTop > 0 ? styles.showButton : styles.hideButton}`}
                         onClick={(_) => {
+                            closeWebSocket();
+                            getOldEntries();
+                        }}>
+                        <img alt="down" src={down} />
+                    </button>
+                    <button type="button"
+                        title="Snap to bottom"
+                        className={`${styles.btnLive} ${isSnappedToBottom && !isWebSocketConnectionClosed ? styles.hideButton : styles.showButton}`}
+                        onClick={(_) => {
+                            reconnectWebSocket();
                             scrollableRef.current.jumpToBottom();
                             setIsSnappedToBottom(true);
                         }}>
@@ -36,7 +135,7 @@ export const EntriesList: React.FC<EntriesListProps> = ({entries, listEntryREF,
                 </div>
 
                 <div className={styles.footer}>
-                    <div>Displaying <b>{entries?.length}</b> results (queried <b>{queriedCurrent}</b>/<b>{queriedTotal}</b>)</div>
+                    <div>Displaying <b>{entries?.length}</b> results out of <b>{queriedTotal}</b> total</div>
                     {startTime !== 0 && <div>Started listening at <span style={{marginRight: 5, fontWeight: 600, fontSize: 13}}>{new Date(startTime).toLocaleString()}</span></div>}
                 </div>
             </div>

ui/src/components/EntryDetailed.tsx

@@ -73,7 +73,7 @@ const EntrySummary: React.FC<any> = ({data, updateQuery}) => {
     const entry = data.base;
 
     return <EntryItem
-        key={entry.id}
+        key={`entry-${entry.id}`}
         entry={entry}
         focusedEntryId={null}
         setFocusedEntryId={null}

ui/src/components/EntryListItem/EntryListItem.tsx

@@ -121,7 +121,7 @@ export const EntryItem: React.FC<EntryProps> = ({entry, focusedEntryId, setFocus
 
     return <>
         <div
-            id={entry.id.toString()}
+            id={`entry-${entry.id.toString()}`}
             className={`${styles.row}
             ${isSelected && !rule && !contractEnabled ? styles.rowSelected : additionalRulesProperties}`}
             onClick={() => {

ui/src/components/Filters.tsx

@@ -13,7 +13,7 @@ interface FiltersProps {
     setQuery: any
     backgroundColor: string
     ws: any
-    openWebSocket: (query: string, resetEntriesBuffer: boolean) => void;
+    openWebSocket: (query: string, resetEntries: boolean) => void;
 }
 
 export const Filters: React.FC<FiltersProps> = ({query, setQuery, backgroundColor, ws, openWebSocket}) => {
@@ -33,7 +33,7 @@ interface QueryFormProps {
     setQuery: any
     backgroundColor: string
     ws: any
-    openWebSocket: (query: string, resetEntriesBuffer: boolean) => void;
+    openWebSocket: (query: string, resetEntries: boolean) => void;
 }
 
 const style = {
@@ -64,7 +64,11 @@ export const QueryForm: React.FC<QueryFormProps> = ({query, setQuery, background
 
     const handleSubmit = (e) => {
         ws.close();
-        openWebSocket(query, true);
+        if (query) {
+            openWebSocket(`(${query}) and leftOff(-1)`, true);
+        } else {
+            openWebSocket(`leftOff(-1)`, true);
+        }
         e.preventDefault();
     }
 

ui/src/components/TrafficPage.tsx

@@ -1,7 +1,6 @@
 import React, {useEffect, useRef, useState} from "react";
 import {Filters} from "./Filters";
 import {EntriesList} from "./EntriesList";
-import {EntryItem} from "./EntryListItem/EntryListItem";
 import {makeStyles} from "@material-ui/core";
 import "./style/TrafficPage.sass";
 import styles from './style/EntriesList.module.sass';
@@ -51,11 +50,12 @@ export const TrafficPage: React.FC<TrafficPageProps> = ({setAnalyzeStatus, onTLS
     const classes = useLayoutStyles();
 
     const [entries, setEntries] = useState([] as any);
-    const [entriesBuffer, setEntriesBuffer] = useState([] as any);
     const [focusedEntryId, setFocusedEntryId] = useState(null);
     const [selectedEntryData, setSelectedEntryData] = useState(null);
     const [connection, setConnection] = useState(ConnectionStatus.Closed);
 
+    const [noMoreDataTop, setNoMoreDataTop] = useState(false);
+
     const [tappingStatus, setTappingStatus] = useState(null);
 
     const [isSnappedToBottom, setIsSnappedToBottom] = useState(true);
@@ -66,7 +66,8 @@ export const TrafficPage: React.FC<TrafficPageProps> = ({setAnalyzeStatus, onTLS
 
     const [queriedCurrent, setQueriedCurrent] = useState(0);
     const [queriedTotal, setQueriedTotal] = useState(0);
-    const [leftOff, setLeftOff] = useState(0);
+    const [leftOffBottom, setLeftOffBottom] = useState(0);
+    const [leftOffTop, setLeftOffTop] = useState(null);
 
     const [startTime, setStartTime] = useState(0);
 
@@ -101,13 +102,13 @@ export const TrafficPage: React.FC<TrafficPageProps> = ({setAnalyzeStatus, onTLS
 
     const listEntry = useRef(null);
 
-    const openWebSocket = (query: string, resetEntriesBuffer: boolean) => {
+    const openWebSocket = (query: string, resetEntries: boolean) => {
-        if (resetEntriesBuffer) {
+        if (resetEntries) {
             setFocusedEntryId(null);
             setEntries([]);
-            setEntriesBuffer([]);
+            setQueriedCurrent(0);
-        } else {
+            setLeftOffTop(null);
-            setEntriesBuffer(entries);
+            setNoMoreDataTop(false);
         }
         ws.current = new WebSocket(MizuWebsocketURL);
         ws.current.onopen = () => {
@@ -120,9 +121,9 @@ export const TrafficPage: React.FC<TrafficPageProps> = ({setAnalyzeStatus, onTLS
         ws.current.onerror = (event) => {
             console.error("WebSocket error:", event);
             if (query) {
-                openWebSocket(`(${query}) and leftOff(${leftOff})`, false);
+                openWebSocket(`(${query}) and leftOff(${leftOffBottom})`, false);
             } else {
-                openWebSocket(`leftOff(${leftOff})`, false);
+                openWebSocket(`leftOff(${leftOffBottom})`, false);
             }
         }
     }
@@ -134,23 +135,14 @@ export const TrafficPage: React.FC<TrafficPageProps> = ({setAnalyzeStatus, onTLS
             switch (message.messageType) {
                 case "entry":
                     const entry = message.data;
-                    var focusThis = false;
+                    if (!focusedEntryId) setFocusedEntryId(entry.id.toString())
-                    if (!focusedEntryId) {
+                    const newEntries = [...entries, entry];
-                        focusThis = true;
+                    if (newEntries.length === 10001) {
-                        setFocusedEntryId(entry.id.toString());
+                        setLeftOffTop(newEntries[0].entry.id);
+                        newEntries.shift();
+                        setNoMoreDataTop(false);
                     }
-                    setEntriesBuffer([
+                    setEntries(newEntries);
-                        ...entriesBuffer,
-                        <EntryItem
-                            key={entry.id}
-                            entry={entry}
-                            focusedEntryId={focusThis ? entry.id.toString() : focusedEntryId}
-                            setFocusedEntryId={setFocusedEntryId}
-                            style={{}}
-                            updateQuery={updateQuery}
-                            headingMode={false}
-                        />
-                    ]);
                     break
                 case "status":
                     setTappingStatus(message.tappingStatus);
@@ -174,24 +166,16 @@ export const TrafficPage: React.FC<TrafficPageProps> = ({setAnalyzeStatus, onTLS
                     });
                     break;
                 case "queryMetadata":
-                    setQueriedCurrent(message.data.current);
+                    setQueriedCurrent(queriedCurrent + message.data.current);
                     setQueriedTotal(message.data.total);
-                    setLeftOff(message.data.leftOff);
+                    setLeftOffBottom(message.data.leftOff);
-                    setEntries(entriesBuffer);
+                    if (leftOffTop === null) {
+                        setLeftOffTop(message.data.leftOff - 1);
+                    }
                     break;
                 case "startTime":
                     setStartTime(message.data);
                     break;
-                case "focusEntry":
-                    // To achieve selecting only one entry, render all elements in the buffer
-                    // with the current `focusedEntryId` value.
-                    entriesBuffer.forEach((entry: any, i: number) => {
-                        entriesBuffer[i] = React.cloneElement(entry, {
-                            focusedEntryId: focusedEntryId
-                        });
-                    })
-                    setEntries(entriesBuffer);
-                    break;
                 default:
                     console.error(`unsupported websocket message type, Got: ${message.messageType}`)
             }
@@ -217,11 +201,6 @@ export const TrafficPage: React.FC<TrafficPageProps> = ({setAnalyzeStatus, onTLS
     useEffect(() => {
         if (!focusedEntryId) return;
         setSelectedEntryData(null);
-
-        if (ws.current.readyState === WebSocket.OPEN) {
-            ws.current.send(focusedEntryId);
-        }
-
         (async () => {
             try {
                 const entryData = await api.getEntry(focusedEntryId);
@@ -241,18 +220,27 @@ export const TrafficPage: React.FC<TrafficPageProps> = ({setAnalyzeStatus, onTLS
                 }
                 console.error(error);
             }
-        })()
+        })();
-    }, [focusedEntryId])
+        // eslint-disable-next-line
+    }, [focusedEntryId]);
+
+    const closeWebSocket = () => {
+        ws.current.close();
+    }
+
+    const reconnectWebSocket = () => {
+        if (query) {
+            openWebSocket(`(${query}) and leftOff(${leftOffBottom})`, false);
+        } else {
+            openWebSocket(`leftOff(${leftOffBottom})`, false);
+        }
+    }
 
     const toggleConnection = () => {
         if (connection === ConnectionStatus.Connected) {
-            ws.current.close();
+            closeWebSocket();
         } else {
-            if (query) {
+            reconnectWebSocket();
-                openWebSocket(`(${query}) and leftOff(${leftOff})`, false);
-            } else {
-                openWebSocket(`leftOff(${leftOff})`, false);
-            }
         }
     }
 
@@ -276,7 +264,10 @@ export const TrafficPage: React.FC<TrafficPageProps> = ({setAnalyzeStatus, onTLS
     }
 
     const onSnapBrokenEvent = () => {
-        setIsSnappedToBottom(false)
+        setIsSnappedToBottom(false);
+        if (connection === ConnectionStatus.Connected) {
+            closeWebSocket();
+        }
     }
 
     return (
@@ -305,13 +296,26 @@ export const TrafficPage: React.FC<TrafficPageProps> = ({setAnalyzeStatus, onTLS
                     <div className={styles.container}>
                         <EntriesList
                             entries={entries}
+                            setEntries={setEntries}
+                            query={query}
                             listEntryREF={listEntry}
                             onSnapBrokenEvent={onSnapBrokenEvent}
                             isSnappedToBottom={isSnappedToBottom}
                             setIsSnappedToBottom={setIsSnappedToBottom}
                             queriedCurrent={queriedCurrent}
+                            setQueriedCurrent={setQueriedCurrent}
                             queriedTotal={queriedTotal}
                             startTime={startTime}
+                            noMoreDataTop={noMoreDataTop}
+                            setNoMoreDataTop={setNoMoreDataTop}
+                            focusedEntryId={focusedEntryId}
+                            setFocusedEntryId={setFocusedEntryId}
+                            updateQuery={updateQuery}
+                            leftOffTop={leftOffTop}
+                            setLeftOffTop={setLeftOffTop}
+                            reconnectWebSocket={reconnectWebSocket}
+                            isWebSocketConnectionClosed={connection === ConnectionStatus.Closed}
+                            closeWebSocket={closeWebSocket}
                         />
                     </div>
                 </div>

ui/src/components/style/EntriesList.module.sass

@@ -38,6 +38,31 @@
   border: 1px solid #627ef7
   background-color: rgba(255, 255, 255, 0.06)
 
+.spinnerContainer
+  display: flex
+  justify-content: center
+  margin-bottom: 10px
+
+.noMoreDataAvailable
+  text-align: center
+  font-weight: 600
+  color: $secondary-font-color
+
+.btnOld
+  position: absolute
+  top: 20px
+  right: 10px
+  background: #205CF5
+  border-radius: 50%
+  height: 35px
+  width: 35px
+  border: none
+  cursor: pointer
+  z-index: 1
+  img
+    height: 10px
+    transform: scaleY(-1)
+
 .btnLive
   position: absolute
   bottom: 10px

ui/src/helpers/api.js

@@ -38,6 +38,14 @@ export default class Api {
         return response.data;
     }
 
+    fetchEntries = async (leftOff, direction, query, limit, timeoutMs) => {
+        const response = await this.client.get(`/entries/?leftOff=${leftOff}&direction=${direction}&query=${query}&limit=${limit}&timeoutMs=${timeoutMs}`).catch(function (thrown) {
+            console.error(thrown.message);
+            return {};
+        });
+        return response.data;
+    }
+
     getRecentTLSLinks = async () => {
         const response = await this.client.get("/status/recentTLSLinks");
         return response.data;