github/kubeshark
1
0
Fork 0
mirror of https://github.com/kubeshark/kubeshark.git synced 2025-07-03 19:40:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add option to limit example len (#1145)

Browse Source 
Do not save OAS example response if it is over the limit.
SpecGen.MaxExampleLen is measured in bytes.
Limit=0 means "don't record any examples", while limit=-1 means "record all".

Limit is configured via the config file / --set: --set oas.max-example-len=100.
The default is 10KB.

This PR breaks the previous config. Moved oas bool (enable/disable OAS) to oas.enable.
Users who wish to disable OAS (or to explicitly enable it) should now to so with --set oas.enable=false instead of --set oas=false.

Co-authored-by: Nimrod Gilboa Markevich <nimrod@up9.com>
Co-authored-by: Nimrod Gilboa Markevich <59927337+nimrod-up9@users.noreply.github.com>
This commit is contained in:
Andrey Pokhilko 2022-06-16 14:18:42 +03:00 committed by GitHub
parent be98d85cb8
commit 7a823e89f1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 48 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
agent/main.go
View File

@ -118,7 +118,7 @@ func hostApi(socketHarOutputChannel chan<- *tapApi.OutputChannelItem) *gin.Engin
api.WebSocketRoutes(ginApp, &eventHandlers)
if config.Config.OAS {
if config.Config.OAS.Enable {
routes.OASRoutes(ginApp)
}
@ -200,7 +200,7 @@ func runInHarReaderMode() {
}
func enableExpFeatureIfNeeded() {
if config.Config.OAS {
if config.Config.OAS.Enable {
oasGenerator := dependency.GetInstance(dependency.OasGeneratorDependency).(oas.OasGenerator)
oasGenerator.Start()
}
@ -227,7 +227,7 @@ func setUIFlags(uiIndexPath string) error {
return err
}
replacedContent := strings.Replace(string(read), "__IS_OAS_ENABLED__", strconv.FormatBool(config.Config.OAS), 1)
replacedContent := strings.Replace(string(read), "__IS_OAS_ENABLED__", strconv.FormatBool(config.Config.OAS.Enable), 1)
replacedContent = strings.Replace(replacedContent, "__IS_SERVICE_MAP_ENABLED__", strconv.FormatBool(config.Config.ServiceMap), 1)
err = ioutil.WriteFile(uiIndexPath, []byte(replacedContent), 0)
@ -363,7 +363,7 @@ func handleIncomingMessageAsTapper(socketConnection *websocket.Conn) {
func initializeDependencies() {
dependency.RegisterGenerator(dependency.ServiceMapGeneratorDependency, func() interface{} { return servicemap.GetDefaultServiceMapInstance() })
dependency.RegisterGenerator(dependency.OasGeneratorDependency, func() interface{} { return oas.GetDefaultOasGeneratorInstance() })
dependency.RegisterGenerator(dependency.OasGeneratorDependency, func() interface{} { return oas.GetDefaultOasGeneratorInstance(config.Config.OAS.MaxExampleLen) })
dependency.RegisterGenerator(dependency.EntriesInserter, func() interface{} { return api.GetBasenineEntryInserterInstance() })
dependency.RegisterGenerator(dependency.EntriesProvider, func() interface{} { return &entries.BasenineEntriesProvider{} })
dependency.RegisterGenerator(dependency.EntriesSocketStreamer, func() interface{} { return &api.BasenineEntryStreamer{} })

6
agent/pkg/controllers/oas_controller_test.go
View File

@ -34,12 +34,12 @@ func TestGetOASSpec(t *testing.T) {
func getRecorderAndContext() (*httptest.ResponseRecorder, *gin.Context) {
dependency.RegisterGenerator(dependency.OasGeneratorDependency, func() interface{} {
return oas.GetDefaultOasGeneratorInstance()
return oas.GetDefaultOasGeneratorInstance(-1)
})
recorder := httptest.NewRecorder()
c, _ := gin.CreateTestContext(recorder)
oas.GetDefaultOasGeneratorInstance().Start()
oas.GetDefaultOasGeneratorInstance().GetServiceSpecs().Store("some", oas.NewGen("some"))
oas.GetDefaultOasGeneratorInstance(-1).Start()
oas.GetDefaultOasGeneratorInstance(-1).GetServiceSpecs().Store("some", oas.NewGen("some"))
return recorder, c
}

2
agent/pkg/oas/ignores.go
View File

@ -9,7 +9,7 @@ var ignoredCtypes = []string{"application/javascript", "application/x-javascript
var ignoredHeaders = []string{
"a-im", "accept",
"authorization", "cache-control", "connection", "content-encoding", "content-length", "content-type", "cookie",
"authorization", "cache-control", "connection", "content-encoding", "content-length", "content-range", "content-type", "cookie",
"date", "dnt", "expect", "forwarded", "from", "front-end-https", "host", "http2-settings",
"max-forwards", "origin", "pragma", "proxy-authorization", "proxy-connection", "range", "referer",
"save-data", "te", "trailer", "transfer-encoding", "upgrade", "upgrade-insecure-requests", "x-download-options",

17
agent/pkg/oas/oas_generator.go
View File

@ -28,13 +28,14 @@ type OasGenerator interface {
}
type defaultOasGenerator struct {
started bool
serviceSpecs *sync.Map
started bool
serviceSpecs *sync.Map
maxExampleLen int
}
func GetDefaultOasGeneratorInstance() *defaultOasGenerator {
func GetDefaultOasGeneratorInstance(maxExampleLen int) *defaultOasGenerator {
syncOnce.Do(func() {
instance = NewDefaultOasGenerator()
instance = NewDefaultOasGenerator(maxExampleLen)
logger.Log.Debug("OAS Generator Initialized")
})
return instance
@ -117,6 +118,7 @@ func (g *defaultOasGenerator) getGen(dest string, urlStr string) *SpecGen {
var gen *SpecGen
if !found {
gen = NewGen(u.Scheme + "://" + dest)
gen.MaxExampleLen = g.maxExampleLen
g.serviceSpecs.Store(dest, gen)
} else {
gen = val.(*SpecGen)
@ -132,9 +134,10 @@ func (g *defaultOasGenerator) GetServiceSpecs() *sync.Map {
return g.serviceSpecs
}
func NewDefaultOasGenerator() *defaultOasGenerator {
func NewDefaultOasGenerator(maxExampleLen int) *defaultOasGenerator {
return &defaultOasGenerator{
started: false,
serviceSpecs: &sync.Map{},
started: false,
serviceSpecs: &sync.Map{},
maxExampleLen: maxExampleLen,
}
}

2
agent/pkg/oas/oas_generator_test.go
View File

@ -8,7 +8,7 @@ import (
)
func TestOASGen(t *testing.T) {
gen := GetDefaultOasGeneratorInstance()
gen := GetDefaultOasGeneratorInstance(-1)
e := new(har.Entry)
err := json.Unmarshal([]byte(`{"startedDateTime": "20000101","request": {"url": "https://host/path", "method": "GET"}, "response": {"status": 200}}`), e)

28
agent/pkg/oas/specgen.go
View File

@ -42,6 +42,8 @@ type reqResp struct { // hello, generics in Go
}
type SpecGen struct {
MaxExampleLen int // -1 unlimited, 0 and above sets limit
oas *openapi.OpenAPI
tree *Node
lock sync.Mutex
@ -59,7 +61,11 @@ func NewGen(server string) *SpecGen {
spec.Servers = make([]*openapi.Server, 0)
spec.Servers = append(spec.Servers, &openapi.Server{URL: server})
gen := SpecGen{oas: spec, tree: new(Node)}
gen := SpecGen{
oas: spec,
tree: new(Node),
MaxExampleLen: -1,
}
return &gen
}
@ -228,7 +234,7 @@ func (g *SpecGen) handlePathObj(entryWithSource *EntryWithSource) (string, error
split = strings.Split(urlParsed.Path, "/")
}
node := g.tree.getOrSet(split, new(openapi.PathObj), entryWithSource.Id)
opObj, err := handleOpObj(entryWithSource, node.pathObj)
opObj, err := handleOpObj(entryWithSource, node.pathObj, g.MaxExampleLen)
if opObj != nil {
return opObj.OperationID, err
@ -237,7 +243,7 @@ func (g *SpecGen) handlePathObj(entryWithSource *EntryWithSource) (string, error
return "", err
}
func handleOpObj(entryWithSource *EntryWithSource, pathObj *openapi.PathObj) (*openapi.Operation, error) {
func handleOpObj(entryWithSource *EntryWithSource, pathObj *openapi.PathObj, limit int) (*openapi.Operation, error) {
entry := entryWithSource.Entry
isSuccess := 100 <= entry.Response.Status && entry.Response.Status < 400
opObj, wasMissing, err := getOpObj(pathObj, entry.Request.Method, isSuccess)
@ -250,12 +256,12 @@ func handleOpObj(entryWithSource *EntryWithSource, pathObj *openapi.PathObj) (*o
return nil, nil
}
err = handleRequest(&entry.Request, opObj, isSuccess, entryWithSource.Id)
err = handleRequest(&entry.Request, opObj, isSuccess, entryWithSource.Id, limit)
if err != nil {
return nil, err
}
err = handleResponse(&entry.Response, opObj, isSuccess, entryWithSource.Id)
err = handleResponse(&entry.Response, opObj, isSuccess, entryWithSource.Id, limit)
if err != nil {
return nil, err
}
@ -342,7 +348,7 @@ func handleCounters(opObj *openapi.Operation, success bool, entryWithSource *Ent
return nil
}
func handleRequest(req *har.Request, opObj *openapi.Operation, isSuccess bool, sampleId string) error {
func handleRequest(req *har.Request, opObj *openapi.Operation, isSuccess bool, sampleId string, limit int) error {
// TODO: we don't handle the situation when header/qstr param can be defined on pathObj level. Also the path param defined on opObj
urlParsed, err := url.Parse(req.URL)
if err != nil {
@ -390,7 +396,7 @@ func handleRequest(req *har.Request, opObj *openapi.Operation, isSuccess bool, s
} else {
reqCtype, _ := getReqCtype(req)
reqMedia, err := fillContent(reqResp{Req: req}, reqBody.Content, reqCtype, sampleId)
reqMedia, err := fillContent(reqResp{Req: req}, reqBody.Content, reqCtype, sampleId, limit)
if err != nil {
return err
}
@ -402,7 +408,7 @@ func handleRequest(req *har.Request, opObj *openapi.Operation, isSuccess bool, s
return nil
}
func handleResponse(resp *har.Response, opObj *openapi.Operation, isSuccess bool, sampleId string) error {
func handleResponse(resp *har.Response, opObj *openapi.Operation, isSuccess bool, sampleId string, limit int) error {
// TODO: we don't support "default" response
respObj, err := getResponseObj(resp, opObj, isSuccess)
if err != nil {
@ -415,7 +421,7 @@ func handleResponse(resp *har.Response, opObj *openapi.Operation, isSuccess bool
respCtype := getRespCtype(resp)
respContent := respObj.Content
respMedia, err := fillContent(reqResp{Resp: resp}, respContent, respCtype, sampleId)
respMedia, err := fillContent(reqResp{Resp: resp}, respContent, respCtype, sampleId, limit)
if err != nil {
return err
}
@ -467,7 +473,7 @@ func handleRespHeaders(reqHeaders []har.Header, respObj *openapi.ResponseObj, sa
}
}
func fillContent(reqResp reqResp, respContent openapi.Content, ctype string, sampleId string) (*openapi.MediaType, error) {
func fillContent(reqResp reqResp, respContent openapi.Content, ctype string, sampleId string, limit int) (*openapi.MediaType, error) {
content, found := respContent[ctype]
if !found {
respContent[ctype] = &openapi.MediaType{}
@ -510,7 +516,7 @@ func fillContent(reqResp reqResp, respContent openapi.Content, ctype string, sam
handleFormDataMultipart(text, content, params)
}
if content.Example == nil && len(exampleMsg) > len(content.Example) {
if len(exampleMsg) > len(content.Example) && (limit < 0 || len(exampleMsg) <= limit) {
content.Example = exampleMsg
}
}

8
agent/pkg/oas/specgen_test.go
View File

@ -48,7 +48,7 @@ func TestEntries(t *testing.T) {
t.FailNow()
}
gen := NewDefaultOasGenerator()
gen := NewDefaultOasGenerator(-1)
gen.serviceSpecs = new(sync.Map)
loadStartingOAS("test_artifacts/catalogue.json", "catalogue", gen.serviceSpecs)
loadStartingOAS("test_artifacts/trcc.json", "trcc-api-service", gen.serviceSpecs)
@ -122,7 +122,7 @@ func TestEntries(t *testing.T) {
}
func TestFileSingle(t *testing.T) {
gen := NewDefaultOasGenerator()
gen := NewDefaultOasGenerator(-1)
gen.serviceSpecs = new(sync.Map)
// loadStartingOAS()
file := "test_artifacts/params.har"
@ -212,7 +212,7 @@ func loadStartingOAS(file string, label string, specs *sync.Map) {
}
func TestEntriesNegative(t *testing.T) {
gen := NewDefaultOasGenerator()
gen := NewDefaultOasGenerator(-1)
gen.serviceSpecs = new(sync.Map)
files := []string{"invalid"}
_, err := feedEntries(files, false, gen)
@ -223,7 +223,7 @@ func TestEntriesNegative(t *testing.T) {
}
func TestEntriesPositive(t *testing.T) {
gen := NewDefaultOasGenerator()
gen := NewDefaultOasGenerator(-1)
gen.serviceSpecs = new(sync.Map)
files := []string{"test_artifacts/params.har"}
_, err := feedEntries(files, false, gen)

2
agent/pkg/oas/test_artifacts/params.har.spec.json
View File

@ -333,7 +333,7 @@
}
}
},
"example": "agent-id=ade\u0026callback-url=\u0026token=sometoken",
"example": "agent-id=ade\u0026callback-url=\u0026token=sometoken-second-val\u0026optional=another",
"x-sample-entry": "000000000000000000000008"
}
},

2
cli/config/configStruct.go
View File

@ -39,7 +39,7 @@ type ConfigStruct struct {
HeadlessMode bool `yaml:"headless" default:"false"`
LogLevelStr string `yaml:"log-level,omitempty" default:"INFO" readonly:""`
ServiceMap bool `yaml:"service-map" default:"true"`
OAS bool `yaml:"oas" default:"true"`
OAS shared.OASConfig `yaml:"oas"`
}
func (config *ConfigStruct) validate() error {

7
shared/models.go
View File

@ -32,6 +32,11 @@ type Resources struct {
MemoryRequests string `yaml:"memory-requests" default:"50Mi"`
}
type OASConfig struct {
Enable bool `yaml:"enabled" default:"true"`
MaxExampleLen int `yaml:"max-example-len" default:"10240"`
}
type MizuAgentConfig struct {
MaxDBSizeBytes int64 `json:"maxDBSizeBytes"`
InsertionFilter string `json:"insertionFilter"`
@ -42,7 +47,7 @@ type MizuAgentConfig struct {
MizuResourcesNamespace string `json:"mizuResourceNamespace"`
AgentDatabasePath string `json:"agentDatabasePath"`
ServiceMap bool `json:"serviceMap"`
OAS bool `json:"oas"`
OAS OASConfig `json:"oas"`
Telemetry bool `json:"telemetry"`
}