github/kubeshark
1
0
Fork 0
mirror of https://github.com/kubeshark/kubeshark.git synced 2025-09-06 21:11:11 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fetch N number of records in M milliseconds timeout before streaming the records (#1056)

Browse Source 
* Fetch N number of records in M milliseconds timeout before streaming the records

* Implement the functionality inside socket data streamer

* Reverse the `fetchData` slice

* #run_acceptance_tests

* Trying to fix the tests.
#run_acceptance_tests

* javascript compilation error.

* #run_acceptance_tests

* Name the method better

* Upgrade Basenine version to `v0.8.0`

* Fix some issues related to `Fetch`

* Upgrade the Basenine version in `Dockerfile` as well

* Remove underscore from the parameter name

* Parameterize fetch timeout ms

Co-authored-by: gadotroee <55343099+gadotroee@users.noreply.github.com>
Co-authored-by: Roee Gadot <roee.gadot@up9.com>
This commit is contained in:
M. Mert Yıldıran
2022-05-09 09:11:27 -07:00
committed by GitHub
parent 4c0aeb8146
commit eef0ee8023
15 changed files with 105 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
agent/go.mod
View File

@@ -20,7 +20,7 @@ require (
github.com/op/go-logging v0.0.0-20160315200505-970db520ece7
github.com/orcaman/concurrent-map v1.0.0
github.com/stretchr/testify v1.7.0
github.com/up9inc/basenine/client/go v0.0.0-20220419100955-e2ca51087607
github.com/up9inc/basenine/client/go v0.0.0-20220508080324-c66c4e1b9337
github.com/up9inc/mizu/logger v0.0.0
github.com/up9inc/mizu/shared v0.0.0
github.com/up9inc/mizu/tap v0.0.0

4
agent/go.sum
View File

@@ -683,8 +683,8 @@ github.com/ugorji/go v1.2.6/go.mod h1:anCg0y61KIhDlPZmnH+so+RQbysYVyDko0IMgJv0Nn
github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
github.com/ugorji/go/codec v1.2.6 h1:7kbGefxLoDBuYXOms4yD7223OpNMMPNPZxXk5TvFcyQ=
github.com/ugorji/go/codec v1.2.6/go.mod h1:V6TCNZ4PHqoHGFZuSG1W8nrCzzdgA2DozYxWFFpvxTw=
github.com/up9inc/basenine/client/go v0.0.0-20220419100955-e2ca51087607 h1:UqxUSkOYOmsLZWQtMSk02ttnhdRwBRLOLt2aDiS9tEk=
github.com/up9inc/basenine/client/go v0.0.0-20220419100955-e2ca51087607/go.mod h1:SvJGPoa/6erhUQV7kvHBwM/0x5LyO6XaG2lUaCaKiUI=
github.com/up9inc/basenine/client/go v0.0.0-20220508080324-c66c4e1b9337 h1:eRXRZnojrZyhbiSuGHl0EPvFtWvx1ZMrsY/bSoBzYNE=
github.com/up9inc/basenine/client/go v0.0.0-20220508080324-c66c4e1b9337/go.mod h1:SvJGPoa/6erhUQV7kvHBwM/0x5LyO6XaG2lUaCaKiUI=
github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74 h1:gga7acRE695APm9hlsSMoOoE65U4/TcqNj90mc69Rlg=
github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0=
github.com/wI2L/jsondiff v0.1.1 h1:r2TkoEet7E4JMO5+s1RCY2R0LrNPNHY6hbDeow2hRHw=

66
agent/pkg/api/socket_data_streamer.go
View File

@@ -3,6 +3,7 @@ package api
import (
"context"
"encoding/json"
"time"
basenine "github.com/up9inc/basenine/client/go"
"github.com/up9inc/mizu/agent/pkg/dependency"
@@ -38,6 +39,11 @@ func (e *BasenineEntryStreamer) Get(ctx context.Context, socketId int, params *W
entryStreamerSocketConnector.SendToastError(socketId, err)
}
leftOff, err := e.fetch(socketId, params, entryStreamerSocketConnector)
if err != nil {
logger.Log.Errorf("Fetch error: %v", err.Error())
}
handleDataChannel := func(c *basenine.Connection, data chan []byte) {
for {
bytes := <-data
@@ -79,7 +85,7 @@ func (e *BasenineEntryStreamer) Get(ctx context.Context, socketId int, params *W
go handleDataChannel(connection, data)
go handleMetaChannel(connection, meta)
if err = connection.Query(query, data, meta); err != nil {
if err = connection.Query(leftOff, query, data, meta); err != nil {
logger.Log.Errorf("Query mode call failed: %v", err)
entryStreamerSocketConnector.CleanupSocket(socketId)
return err
@@ -94,3 +100,61 @@ func (e *BasenineEntryStreamer) Get(ctx context.Context, socketId int, params *W
return nil
}
// Reverses a []byte slice.
func (e *BasenineEntryStreamer) fetch(socketId int, params *WebSocketParams, connector EntryStreamerSocketConnector) (leftOff string, err error) {
if params.Fetch <= 0 {
leftOff = params.LeftOff
return
}
var data [][]byte
var firstMeta []byte
var lastMeta []byte
data, firstMeta, lastMeta, err = basenine.Fetch(
shared.BasenineHost,
shared.BaseninePort,
params.LeftOff,
-1,
params.Query,
params.Fetch,
time.Duration(params.TimeoutMs)*time.Millisecond,
)
if err != nil {
return
}
var firstMetadata *basenine.Metadata
err = json.Unmarshal(firstMeta, &firstMetadata)
if err != nil {
return
}
leftOff = firstMetadata.LeftOff
var lastMetadata *basenine.Metadata
err = json.Unmarshal(lastMeta, &lastMetadata)
if err != nil {
return
}
connector.SendMetadata(socketId, lastMetadata)
data = e.reverseBytesSlice(data)
for _, row := range data {
var entry *tapApi.Entry
err = json.Unmarshal(row, &entry)
if err != nil {
break
}
connector.SendEntry(socketId, entry, params)
}
return
}
// Reverses a []byte slice.
func (e *BasenineEntryStreamer) reverseBytesSlice(arr [][]byte) (newArr [][]byte) {
for i := len(arr) - 1; i >= 0; i-- {
newArr = append(newArr, arr[i])
}
return newArr
}

3
agent/pkg/api/socket_routes.go
View File

@@ -34,8 +34,11 @@ type SocketConnection struct {
}
type WebSocketParams struct {
LeftOff string `json:"leftOff"`
Query string `json:"query"`
EnableFullEntries bool `json:"enableFullEntries"`
Fetch int `json:"fetch"`
TimeoutMs int `json:"timeoutMs"`
}
var (

4
agent/pkg/entries/entries_provider.go
View File

@@ -22,7 +22,7 @@ type EntriesProvider interface {
type BasenineEntriesProvider struct{}
func (e *BasenineEntriesProvider) GetEntries(entriesRequest *models.EntriesRequest) ([]*tapApi.EntryWrapper, *basenine.Metadata, error) {
data, meta, err := basenine.Fetch(shared.BasenineHost, shared.BaseninePort,
data, _, lastMeta, err := basenine.Fetch(shared.BasenineHost, shared.BaseninePort,
entriesRequest.LeftOff, entriesRequest.Direction, entriesRequest.Query,
entriesRequest.Limit, time.Duration(entriesRequest.TimeoutMs)*time.Millisecond)
if err != nil {
@@ -49,7 +49,7 @@ func (e *BasenineEntriesProvider) GetEntries(entriesRequest *models.EntriesReque
}
var metadata *basenine.Metadata
err = json.Unmarshal(meta, &metadata)
err = json.Unmarshal(lastMeta, &metadata)
if err != nil {
logger.Log.Debugf("Error recieving metadata: %v", err.Error())
}

2
agent/pkg/oas/oas_generator.go
View File

@@ -104,7 +104,7 @@ func (g *defaultOasGenerator) runGenerator() {
g.dbMutex.Lock()
defer g.dbMutex.Unlock()
logger.Log.Infof("Querying DB for OAS generator with query '%s'", g.entriesQuery)
if err := g.dbConn.Query(g.entriesQuery, dataChan, metaChan); err != nil {
if err := g.dbConn.Query("", g.entriesQuery, dataChan, metaChan); err != nil {
logger.Log.Errorf("Query mode call failed: %v", err)
}

2
agent/pkg/up9/main.go
View File

@@ -327,7 +327,7 @@ BasenineReconnect:
go handleMetaChannel(&wg, connection, meta)
wg.Add(2)
if err = connection.Query(query, data, meta); err != nil {
if err = connection.Query("", query, data, meta); err != nil {
logger.Log.Errorf("Query mode call failed: %v", err)
connection.Close()
time.Sleep(shared.BasenineReconnectInterval * time.Second)