diff --git a/Dockerfile b/Dockerfile index 62ba07c93..4923cc9ae 100644 --- a/Dockerfile +++ b/Dockerfile @@ -94,8 +94,8 @@ RUN go build -ldflags="-extldflags=-static -s -w \ -X 'github.com/up9inc/mizu/agent/pkg/version.Ver=${VER}'" -o mizuagent . # Download Basenine executable, verify the sha1sum -ADD https://github.com/up9inc/basenine/releases/download/v0.7.3/basenine_linux_${GOARCH} ./basenine_linux_${GOARCH} -ADD https://github.com/up9inc/basenine/releases/download/v0.7.3/basenine_linux_${GOARCH}.sha256 ./basenine_linux_${GOARCH}.sha256 +ADD https://github.com/up9inc/basenine/releases/download/v0.8.1/basenine_linux_${GOARCH} ./basenine_linux_${GOARCH} +ADD https://github.com/up9inc/basenine/releases/download/v0.8.1/basenine_linux_${GOARCH}.sha256 ./basenine_linux_${GOARCH}.sha256 RUN shasum -a 256 -c basenine_linux_"${GOARCH}".sha256 && \ chmod +x ./basenine_linux_"${GOARCH}" && \ diff --git a/acceptanceTests/cypress/integration/testHelpers/TrafficHelper.js b/acceptanceTests/cypress/integration/testHelpers/TrafficHelper.js index e63821d9c..1fb286760 100644 --- a/acceptanceTests/cypress/integration/testHelpers/TrafficHelper.js +++ b/acceptanceTests/cypress/integration/testHelpers/TrafficHelper.js @@ -57,13 +57,6 @@ export function rightOnHoverCheck(path, expectedText) { cy.get(`#rightSideContainer [data-cy='QueryableTooltip']`).invoke('text').should('match', new RegExp(expectedText)); } -export function checkThatAllEntriesShown() { - cy.get('#entries-length').then(number => { - if (number.text() === '1') - cy.get('[title="Fetch old records"]').click(); - }); -} - export function checkFilterByMethod(funcDict) { const {protocol, method, methodQuery, summary, summaryQuery} = funcDict; const summaryDict = getSummaryDict(summary, summaryQuery); @@ -76,12 +69,7 @@ export function checkFilterByMethod(funcDict) { cy.get('[type="submit"]').click(); cy.get('.w-tc-editor').should('have.attr', 'style').and('include', Cypress.env('greenFilterColor')); - cy.get('#entries-length').then(number => { - // if the entries list isn't expanded it expands here - if (number.text() === '0' || number.text() === '1') // todo change when TRA-4262 is fixed - cy.get('[title="Fetch old records"]').click(); - - cy.get('#entries-length').should('not.have.text', '0').and('not.have.text', '1').then(() => { + cy.get('#entries-length').should('not.have.text', '0').then(() => { cy.get(`#list [id]`).then(elements => { const listElmWithIdAttr = Object.values(elements); let doneCheckOnFirst = false; @@ -108,7 +96,6 @@ export function checkFilterByMethod(funcDict) { }); }); }); - }); } export function getEntryId(id) { diff --git a/acceptanceTests/cypress/integration/tests/IgnoredUserAgents.js b/acceptanceTests/cypress/integration/tests/IgnoredUserAgents.js index f6c11a85d..50f59953a 100644 --- a/acceptanceTests/cypress/integration/tests/IgnoredUserAgents.js +++ b/acceptanceTests/cypress/integration/tests/IgnoredUserAgents.js @@ -1,5 +1,4 @@ import { - checkThatAllEntriesShown, isValueExistsInElement, resizeToHugeMizu, } from "../testHelpers/TrafficHelper"; @@ -12,13 +11,14 @@ checkEntries(); function checkEntries() { it('checking all entries', function () { - checkThatAllEntriesShown(); - resizeToHugeMizu(); + cy.get('#entries-length').should('not.have.text', '0').then(() => { + resizeToHugeMizu(); - cy.get('#list [id^=entry]').each(entryElement => { - entryElement.click(); - cy.get('#tbody-Headers').should('be.visible'); - isValueExistsInElement(false, 'Ignored-User-Agent', '#tbody-Headers'); + cy.get('#list [id^=entry]').each(entryElement => { + entryElement.click(); + cy.get('#tbody-Headers').should('be.visible'); + isValueExistsInElement(false, 'Ignored-User-Agent', '#tbody-Headers'); + }); }); }); } diff --git a/acceptanceTests/cypress/integration/tests/UiTest.js b/acceptanceTests/cypress/integration/tests/UiTest.js index 2e538ac18..6b9de5069 100644 --- a/acceptanceTests/cypress/integration/tests/UiTest.js +++ b/acceptanceTests/cypress/integration/tests/UiTest.js @@ -218,12 +218,8 @@ function checkFilter(filterDetails) { checkRightSideResponseBody(); }); - cy.get('[title="Fetch old records"]').click(); resizeToHugeMizu(); - // waiting for the entries number to load - cy.get('#entries-length', {timeout: refreshWaitTimeout}).should('have.text', totalEntries); - // checking only 'leftTextCheck' on all entries because the rest of the checks require more time cy.get(`#list [id^=entry]`).each(elem => { const element = elem[0]; diff --git a/agent/go.mod b/agent/go.mod index 9564468c6..37ba205a8 100644 --- a/agent/go.mod +++ b/agent/go.mod @@ -20,7 +20,7 @@ require ( github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 github.com/orcaman/concurrent-map v1.0.0 github.com/stretchr/testify v1.7.0 - github.com/up9inc/basenine/client/go v0.0.0-20220419100955-e2ca51087607 + github.com/up9inc/basenine/client/go v0.0.0-20220508080324-c66c4e1b9337 github.com/up9inc/mizu/logger v0.0.0 github.com/up9inc/mizu/shared v0.0.0 github.com/up9inc/mizu/tap v0.0.0 diff --git a/agent/go.sum b/agent/go.sum index 624535717..a8a3bbdfc 100644 --- a/agent/go.sum +++ b/agent/go.sum @@ -683,8 +683,8 @@ github.com/ugorji/go v1.2.6/go.mod h1:anCg0y61KIhDlPZmnH+so+RQbysYVyDko0IMgJv0Nn github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= github.com/ugorji/go/codec v1.2.6 h1:7kbGefxLoDBuYXOms4yD7223OpNMMPNPZxXk5TvFcyQ= github.com/ugorji/go/codec v1.2.6/go.mod h1:V6TCNZ4PHqoHGFZuSG1W8nrCzzdgA2DozYxWFFpvxTw= -github.com/up9inc/basenine/client/go v0.0.0-20220419100955-e2ca51087607 h1:UqxUSkOYOmsLZWQtMSk02ttnhdRwBRLOLt2aDiS9tEk= -github.com/up9inc/basenine/client/go v0.0.0-20220419100955-e2ca51087607/go.mod h1:SvJGPoa/6erhUQV7kvHBwM/0x5LyO6XaG2lUaCaKiUI= +github.com/up9inc/basenine/client/go v0.0.0-20220508080324-c66c4e1b9337 h1:eRXRZnojrZyhbiSuGHl0EPvFtWvx1ZMrsY/bSoBzYNE= +github.com/up9inc/basenine/client/go v0.0.0-20220508080324-c66c4e1b9337/go.mod h1:SvJGPoa/6erhUQV7kvHBwM/0x5LyO6XaG2lUaCaKiUI= github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74 h1:gga7acRE695APm9hlsSMoOoE65U4/TcqNj90mc69Rlg= github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= github.com/wI2L/jsondiff v0.1.1 h1:r2TkoEet7E4JMO5+s1RCY2R0LrNPNHY6hbDeow2hRHw= diff --git a/agent/pkg/api/socket_data_streamer.go b/agent/pkg/api/socket_data_streamer.go index 42094875d..7ed1d01cb 100644 --- a/agent/pkg/api/socket_data_streamer.go +++ b/agent/pkg/api/socket_data_streamer.go @@ -3,6 +3,7 @@ package api import ( "context" "encoding/json" + "time" basenine "github.com/up9inc/basenine/client/go" "github.com/up9inc/mizu/agent/pkg/dependency" @@ -38,6 +39,11 @@ func (e *BasenineEntryStreamer) Get(ctx context.Context, socketId int, params *W entryStreamerSocketConnector.SendToastError(socketId, err) } + leftOff, err := e.fetch(socketId, params, entryStreamerSocketConnector) + if err != nil { + logger.Log.Errorf("Fetch error: %v", err.Error()) + } + handleDataChannel := func(c *basenine.Connection, data chan []byte) { for { bytes := <-data @@ -79,7 +85,7 @@ func (e *BasenineEntryStreamer) Get(ctx context.Context, socketId int, params *W go handleDataChannel(connection, data) go handleMetaChannel(connection, meta) - if err = connection.Query(query, data, meta); err != nil { + if err = connection.Query(leftOff, query, data, meta); err != nil { logger.Log.Errorf("Query mode call failed: %v", err) entryStreamerSocketConnector.CleanupSocket(socketId) return err @@ -94,3 +100,61 @@ func (e *BasenineEntryStreamer) Get(ctx context.Context, socketId int, params *W return nil } + +// Reverses a []byte slice. +func (e *BasenineEntryStreamer) fetch(socketId int, params *WebSocketParams, connector EntryStreamerSocketConnector) (leftOff string, err error) { + if params.Fetch <= 0 { + leftOff = params.LeftOff + return + } + + var data [][]byte + var firstMeta []byte + var lastMeta []byte + data, firstMeta, lastMeta, err = basenine.Fetch( + shared.BasenineHost, + shared.BaseninePort, + params.LeftOff, + -1, + params.Query, + params.Fetch, + time.Duration(params.TimeoutMs)*time.Millisecond, + ) + if err != nil { + return + } + + var firstMetadata *basenine.Metadata + err = json.Unmarshal(firstMeta, &firstMetadata) + if err != nil { + return + } + leftOff = firstMetadata.LeftOff + + var lastMetadata *basenine.Metadata + err = json.Unmarshal(lastMeta, &lastMetadata) + if err != nil { + return + } + connector.SendMetadata(socketId, lastMetadata) + + data = e.reverseBytesSlice(data) + for _, row := range data { + var entry *tapApi.Entry + err = json.Unmarshal(row, &entry) + if err != nil { + break + } + + connector.SendEntry(socketId, entry, params) + } + return +} + +// Reverses a []byte slice. +func (e *BasenineEntryStreamer) reverseBytesSlice(arr [][]byte) (newArr [][]byte) { + for i := len(arr) - 1; i >= 0; i-- { + newArr = append(newArr, arr[i]) + } + return newArr +} diff --git a/agent/pkg/api/socket_routes.go b/agent/pkg/api/socket_routes.go index 33d428fa0..7e2c1b896 100644 --- a/agent/pkg/api/socket_routes.go +++ b/agent/pkg/api/socket_routes.go @@ -34,8 +34,11 @@ type SocketConnection struct { } type WebSocketParams struct { + LeftOff string `json:"leftOff"` Query string `json:"query"` EnableFullEntries bool `json:"enableFullEntries"` + Fetch int `json:"fetch"` + TimeoutMs int `json:"timeoutMs"` } var ( diff --git a/agent/pkg/entries/entries_provider.go b/agent/pkg/entries/entries_provider.go index ceb8c34a7..f9b97fc17 100644 --- a/agent/pkg/entries/entries_provider.go +++ b/agent/pkg/entries/entries_provider.go @@ -22,7 +22,7 @@ type EntriesProvider interface { type BasenineEntriesProvider struct{} func (e *BasenineEntriesProvider) GetEntries(entriesRequest *models.EntriesRequest) ([]*tapApi.EntryWrapper, *basenine.Metadata, error) { - data, meta, err := basenine.Fetch(shared.BasenineHost, shared.BaseninePort, + data, _, lastMeta, err := basenine.Fetch(shared.BasenineHost, shared.BaseninePort, entriesRequest.LeftOff, entriesRequest.Direction, entriesRequest.Query, entriesRequest.Limit, time.Duration(entriesRequest.TimeoutMs)*time.Millisecond) if err != nil { @@ -49,7 +49,7 @@ func (e *BasenineEntriesProvider) GetEntries(entriesRequest *models.EntriesReque } var metadata *basenine.Metadata - err = json.Unmarshal(meta, &metadata) + err = json.Unmarshal(lastMeta, &metadata) if err != nil { logger.Log.Debugf("Error recieving metadata: %v", err.Error()) } diff --git a/agent/pkg/oas/oas_generator.go b/agent/pkg/oas/oas_generator.go index 5ca61a9dc..5132d6e21 100644 --- a/agent/pkg/oas/oas_generator.go +++ b/agent/pkg/oas/oas_generator.go @@ -104,7 +104,7 @@ func (g *defaultOasGenerator) runGenerator() { g.dbMutex.Lock() defer g.dbMutex.Unlock() logger.Log.Infof("Querying DB for OAS generator with query '%s'", g.entriesQuery) - if err := g.dbConn.Query(g.entriesQuery, dataChan, metaChan); err != nil { + if err := g.dbConn.Query("", g.entriesQuery, dataChan, metaChan); err != nil { logger.Log.Errorf("Query mode call failed: %v", err) } diff --git a/agent/pkg/up9/main.go b/agent/pkg/up9/main.go index 513ed8216..bdd44b476 100644 --- a/agent/pkg/up9/main.go +++ b/agent/pkg/up9/main.go @@ -327,7 +327,7 @@ BasenineReconnect: go handleMetaChannel(&wg, connection, meta) wg.Add(2) - if err = connection.Query(query, data, meta); err != nil { + if err = connection.Query("", query, data, meta); err != nil { logger.Log.Errorf("Query mode call failed: %v", err) connection.Close() time.Sleep(shared.BasenineReconnectInterval * time.Second) diff --git a/ui-common/src/components/TrafficViewer/EntriesList.tsx b/ui-common/src/components/TrafficViewer/EntriesList.tsx index f14c8bd90..9c03cc58b 100644 --- a/ui-common/src/components/TrafficViewer/EntriesList.tsx +++ b/ui-common/src/components/TrafficViewer/EntriesList.tsx @@ -23,7 +23,7 @@ interface EntriesListProps { setIsSnappedToBottom: any; noMoreDataTop: boolean; setNoMoreDataTop: (flag: boolean) => void; - openWebSocket: (query: string, resetEntries: boolean) => void; + openWebSocket: (leftOff: string, query: string, resetEntries: boolean, fetch: number, fetchTimeoutMs: number) => void; scrollableRef: any; ws: any; } @@ -195,11 +195,7 @@ export const EntriesList: React.FC = ({ className={`${styles.btnLive} ${isSnappedToBottom && !isWsConnectionClosed ? styles.hideButton : styles.showButton}`} onClick={(_) => { if (isWsConnectionClosed) { - if (query) { - openWebSocket(`(${query}) and leftOff("${leftOffBottom}")`, false); - } else { - openWebSocket(`leftOff("${leftOffBottom}")`, false); - } + openWebSocket(leftOffBottom, query, false, 0, 0); } scrollableRef.current.jumpToBottom(); setIsSnappedToBottom(true); diff --git a/ui-common/src/components/TrafficViewer/TrafficViewer.tsx b/ui-common/src/components/TrafficViewer/TrafficViewer.tsx index a8a0e7380..f1eef2124 100644 --- a/ui-common/src/components/TrafficViewer/TrafficViewer.tsx +++ b/ui-common/src/components/TrafficViewer/TrafficViewer.tsx @@ -20,7 +20,7 @@ import {StatusBar} from "../UI/StatusBar"; import tappingStatusAtom from "../../recoil/tappingStatus/atom"; import {TOAST_CONTAINER_ID} from "../../configs/Consts"; import leftOffTopAtom from "../../recoil/leftOffTop"; -import { DEFAULT_QUERY } from '../../hooks/useWS'; +import { DEFAULT_LEFTOFF, DEFAULT_FETCH, DEFAULT_FETCH_TIMEOUT_MS } from '../../hooks/useWS'; const useLayoutStyles = makeStyles(() => ({ details: { @@ -114,11 +114,7 @@ export const TrafficViewer: React.FC = ({ const ws = useRef(null); const openEmptyWebSocket = () => { - if (query) { - openWebSocket(`(${query}) and ${DEFAULT_QUERY}`, true); - } else { - openWebSocket(DEFAULT_QUERY, true); - } + openWebSocket(DEFAULT_LEFTOFF, query, true, DEFAULT_FETCH, DEFAULT_FETCH_TIMEOUT_MS); } const closeWebSocket = () => { @@ -129,7 +125,7 @@ export const TrafficViewer: React.FC = ({ } const listEntry = useRef(null); - const openWebSocket = (query: string, resetEntries: boolean) => { + const openWebSocket = (leftOff: string, query: string, resetEntries: boolean, fetch: number, fetchTimeoutMs: number) => { if (resetEntries) { setFocusedEntryId(null); setEntries([]); @@ -138,7 +134,7 @@ export const TrafficViewer: React.FC = ({ } try { ws.current = new WebSocket(webSocketUrl); - sendQueryWhenWsOpen(query); + sendQueryWhenWsOpen(leftOff, query, fetch, fetchTimeoutMs); ws.current.onopen = () => { setWsReadyState(ws?.current?.readyState); @@ -157,12 +153,18 @@ export const TrafficViewer: React.FC = ({ } } - const sendQueryWhenWsOpen = (query) => { + const sendQueryWhenWsOpen = (leftOff: string, query: string, fetch: number, fetchTimeoutMs: number) => { setTimeout(() => { if (ws?.current?.readyState === WebSocket.OPEN) { - ws.current.send(JSON.stringify({"query": query, "enableFullEntries": false})); + ws.current.send(JSON.stringify({ + "leftOff": leftOff, + "query": query, + "enableFullEntries": false, + "fetch": fetch, + "timeoutMs": fetchTimeoutMs + })); } else { - sendQueryWhenWsOpen(query); + sendQueryWhenWsOpen(leftOff, query, fetch, fetchTimeoutMs); } }, 500) } diff --git a/ui-common/src/hooks/useWS.tsx b/ui-common/src/hooks/useWS.tsx index a49c8760c..f06ea280d 100644 --- a/ui-common/src/hooks/useWS.tsx +++ b/ui-common/src/hooks/useWS.tsx @@ -7,7 +7,9 @@ enum WebSocketReadyState { CLOSED } -export const DEFAULT_QUERY = `leftOff("latest")`; +export const DEFAULT_LEFTOFF = `latest`; +export const DEFAULT_FETCH = 50; +export const DEFAULT_FETCH_TIMEOUT_MS = 3000; const useWS = (wsUrl: string) => { const [message, setMessage] = useState(null); diff --git a/ui-common/src/index.tsx b/ui-common/src/index.tsx index 08db78d49..d0b4eb13e 100644 --- a/ui-common/src/index.tsx +++ b/ui-common/src/index.tsx @@ -1,11 +1,11 @@ import TrafficViewer from './components/TrafficViewer/TrafficViewer'; import * as UI from "./components/UI" import { StatusBar } from './components/UI'; -import useWS, { DEFAULT_QUERY } from './hooks/useWS'; +import useWS, { DEFAULT_LEFTOFF } from './hooks/useWS'; import { AnalyzeButton } from "./components/AnalyzeButton/AnalyzeButton" import OasModal from './components/OasModal/OasModal'; import { ServiceMapModal } from './components/ServiceMapModal/ServiceMapModal'; export { UI, AnalyzeButton, StatusBar, OasModal, ServiceMapModal } -export { useWS, DEFAULT_QUERY } +export { useWS, DEFAULT_LEFTOFF } export default TrafficViewer;