github/kubeshark
1
0
Fork 0
mirror of https://github.com/kubeshark/kubeshark.git synced 2025-06-24 07:14:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

OAS: Some minor fixes (#762)

Browse Source 
* Commit some fixes

* Accept expected

Co-authored-by: Igor Gov <iggvrv@gmail.com>
This commit is contained in:
Andrey Pokhilko 2022-02-07 10:12:05 +03:00 committed by GitHub
parent ea678a4d8c
commit 0f77cf8ef9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 120 additions and 72 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
agent/pkg/api/main.go
View File

@ -140,7 +140,7 @@ func startReadingChannel(outputItems <-chan *tapApi.OutputChannelItem, extension
mizuEntry.Rules = rules mizuEntry.Rules = rules
} }
entryWSource := oas.EntryWithSource{Entry: *harEntry, Source: mizuEntry.Source.Name} entryWSource := oas.EntryWithSource{Entry: *harEntry, Source: mizuEntry.Source.Name, Id: mizuEntry.Id}
oas.GetOasGeneratorInstance().PushEntry(&entryWSource) oas.GetOasGeneratorInstance().PushEntry(&entryWSource)
} }

4
agent/pkg/oas/counters.go
View File

@ -17,6 +17,10 @@ type Counter struct {
} }
func (c *Counter) addEntry(ts float64, rt float64, succ bool, dur float64) { func (c *Counter) addEntry(ts float64, rt float64, succ bool, dur float64) {
if dur < 0 {
panic("Duration cannot be negative")
}
c.Entries += 1 c.Entries += 1
c.SumRT += rt c.SumRT += rt
c.SumDuration += dur c.SumDuration += dur

37
agent/pkg/oas/feeder_test.go
View File

@ -21,20 +21,33 @@ func getFiles(baseDir string) (result []string, err error) {
result = make([]string, 0) result = make([]string, 0)
logger.Log.Infof("Reading files from tree: %s", baseDir) logger.Log.Infof("Reading files from tree: %s", baseDir)
inputs := []string{baseDir}
// https://yourbasic.org/golang/list-files-in-directory/ // https://yourbasic.org/golang/list-files-in-directory/
err = filepath.Walk(baseDir, visitor := func(path string, info os.FileInfo, err error) error {
func(path string, info os.FileInfo, err error) error { if err != nil {
if err != nil { return err
return err }
}
ext := strings.ToLower(filepath.Ext(path))
if !info.IsDir() && (ext == ".har" || ext == ".ldjson") {
result = append(result, path)
}
if info.Mode()&os.ModeSymlink != 0 {
path, _ = os.Readlink(path)
inputs = append(inputs, path)
return nil return nil
}) }
ext := strings.ToLower(filepath.Ext(path))
if !info.IsDir() && (ext == ".har" || ext == ".ldjson") {
result = append(result, path)
}
return nil
}
for len(inputs) > 0 {
path := inputs[0]
inputs = inputs[1:]
err = filepath.Walk(path, visitor)
}
sort.SliceStable(result, func(i, j int) bool { sort.SliceStable(result, func(i, j int) bool {
return fileSize(result[i]) < fileSize(result[j]) return fileSize(result[i]) < fileSize(result[j])
@ -126,7 +139,7 @@ func feedEntry(entry *har.Entry, source string, isSync bool, file string) {
logger.Log.Debugf("Interesting: %s", entry.Request.URL) logger.Log.Debugf("Interesting: %s", entry.Request.URL)
} }
ews := EntryWithSource{Entry: *entry, Source: source} ews := EntryWithSource{Entry: *entry, Source: source, Id: uint(0)}
if isSync { if isSync {
GetOasGeneratorInstance().entriesChan <- ews // blocking variant, right? GetOasGeneratorInstance().entriesChan <- ews // blocking variant, right?
} else { } else {

1
agent/pkg/oas/oas_generator.go
View File

@ -107,6 +107,7 @@ func newOasGenerator() *oasGenerator {
type EntryWithSource struct { type EntryWithSource struct {
Source string Source string
Entry har.Entry Entry har.Entry
Id uint
} }
type oasGenerator struct { type oasGenerator struct {

29
agent/pkg/oas/specgen.go
View File

@ -26,6 +26,7 @@ import (
const LastSeenTS = "x-last-seen-ts" const LastSeenTS = "x-last-seen-ts"
const CountersTotal = "x-counters-total" const CountersTotal = "x-counters-total"
const CountersPerSource = "x-counters-per-source" const CountersPerSource = "x-counters-per-source"
const SampleId = "x-sample-entry"
type reqResp struct { // hello, generics in Go type reqResp struct { // hello, generics in Go
Req *har.Request Req *har.Request
@ -306,7 +307,7 @@ func handleCounters(opObj *openapi.Operation, success bool, entryWithSource *Ent
rt := float64(entryWithSource.Entry.Time) / 1000 rt := float64(entryWithSource.Entry.Time) / 1000
dur := 0.0 dur := 0.0
if prevTs != 0 { if prevTs != 0 && ts >= prevTs {
dur = ts - prevTs dur = ts - prevTs
} }
@ -328,19 +329,39 @@ func handleCounters(opObj *openapi.Operation, success bool, entryWithSource *Ent
return err return err
} }
err = opObj.Extensions.SetExtension(SampleId, entryWithSource.Id)
if err != nil {
return err
}
return nil return nil
} }
func handleRequest(req *har.Request, opObj *openapi.Operation, isSuccess bool) error { func handleRequest(req *har.Request, opObj *openapi.Operation, isSuccess bool) error {
// TODO: we don't handle the situation when header/qstr param can be defined on pathObj level. Also the path param defined on opObj // TODO: we don't handle the situation when header/qstr param can be defined on pathObj level. Also the path param defined on opObj
urlParsed, err := url.Parse(req.URL)
if err != nil {
return err
}
qs := make([]har.NVP, 0)
for name, vals := range urlParsed.Query() {
for _, val := range vals {
qs = append(qs, har.NVP{Name: name, Value: val})
}
}
if len(qs) != len(req.QueryString) {
logger.Log.Warningf("QStr params in HAR do not match URL: %s", req.URL)
}
qstrGW := nvParams{ qstrGW := nvParams{
In: openapi.InQuery, In: openapi.InQuery,
Pairs: req.QueryString, Pairs: qs,
IsIgnored: func(name string) bool { return false }, IsIgnored: func(name string) bool { return false },
GeneralizeName: func(name string) string { return name }, GeneralizeName: func(name string) string { return name },
} }
handleNameVals(qstrGW, &opObj.Parameters) handleNameVals(qstrGW, &opObj.Parameters, false)
hdrGW := nvParams{ hdrGW := nvParams{
In: openapi.InHeader, In: openapi.InHeader,
@ -348,7 +369,7 @@ func handleRequest(req *har.Request, opObj *openapi.Operation, isSuccess bool) e
IsIgnored: isHeaderIgnored, IsIgnored: isHeaderIgnored,
GeneralizeName: strings.ToLower, GeneralizeName: strings.ToLower,
} }
handleNameVals(hdrGW, &opObj.Parameters) handleNameVals(hdrGW, &opObj.Parameters, true)
if isSuccess { if isSuccess {
reqBody, err := getRequestBody(req, opObj) reqBody, err := getRequestBody(req, opObj)

115
agent/pkg/oas/test_artifacts/params.har.spec.json
View File

@ -2,7 +2,7 @@
"openapi": "3.1.0", "openapi": "3.1.0",
"info": { "info": {
"title": "https://httpbin.org", "title": "https://httpbin.org",
"description": "Mizu observed 13 entries (0 failed), at 0.154 hits/s, average response time is 0.251 seconds", "description": "Mizu observed 13 entries (0 failed), at 0.155 hits/s, average response time is 0.251 seconds",
"version": "1.0" "version": "1.0"
}, },
"servers": [ "servers": [
@ -15,7 +15,7 @@
"get": { "get": {
"summary": "/appears-once", "summary": "/appears-once",
"description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.630 seconds", "description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.630 seconds",
"operationId": "8dfc5a75-5f9b-4b57-ad88-77154a016201", "operationId": "89aa39f6-78d0-411b-b701-a33bd77868b0",
"responses": { "responses": {
"200": { "200": {
"description": "Successful call with status 200", "description": "Successful call with status 200",
@ -26,6 +26,15 @@
} }
} }
}, },
"x-last-seen-ts": 1567750580.0471218,
"x-counters-total": {
"entries": 1,
"failures": 0,
"firstSeen": 1567750580.0471218,
"lastSeen": 1567750580.0471218,
"sumRT": 0.63,
"sumDuration": 0
},
"x-counters-per-source": { "x-counters-per-source": {
"": { "": {
"entries": 1, "entries": 1,
@ -36,22 +45,14 @@
"sumDuration": 0 "sumDuration": 0
} }
}, },
"x-last-seen-ts": 1567750580.0471218, "x-sample-entry": 0
"x-counters-total": {
"entries": 1,
"failures": 0,
"firstSeen": 1567750580.0471218,
"lastSeen": 1567750580.0471218,
"sumRT": 0.63,
"sumDuration": 0
}
} }
}, },
"/appears-twice": { "/appears-twice": {
"get": { "get": {
"summary": "/appears-twice", "summary": "/appears-twice",
"description": "Mizu observed 2 entries (0 failed), at 0.500 hits/s, average response time is 0.630 seconds", "description": "Mizu observed 2 entries (0 failed), at 0.500 hits/s, average response time is 0.630 seconds",
"operationId": "844ae464-fcee-4bd7-97cf-aaa096196c02", "operationId": "f5e2b5a2-e01a-45f4-bde1-15a7e7a06d3c",
"responses": { "responses": {
"200": { "200": {
"description": "Successful call with status 200", "description": "Successful call with status 200",
@ -62,6 +63,14 @@
} }
} }
}, },
"x-counters-total": {
"entries": 2,
"failures": 0,
"firstSeen": 1567750580.7471218,
"lastSeen": 1567750581.7471218,
"sumRT": 1.26,
"sumDuration": 1
},
"x-counters-per-source": { "x-counters-per-source": {
"": { "": {
"entries": 2, "entries": 2,
@ -72,22 +81,15 @@
"sumDuration": 1 "sumDuration": 1
} }
}, },
"x-last-seen-ts": 1567750581.7471218, "x-sample-entry": 0,
"x-counters-total": { "x-last-seen-ts": 1567750581.7471218
"entries": 2,
"failures": 0,
"firstSeen": 1567750580.7471218,
"lastSeen": 1567750581.7471218,
"sumRT": 1.26,
"sumDuration": 1
}
} }
}, },
"/body-optional": { "/body-optional": {
"post": { "post": {
"summary": "/body-optional", "summary": "/body-optional",
"description": "Mizu observed 3 entries (0 failed), at 0.000 hits/s, average response time is 0.001 seconds", "description": "Mizu observed 3 entries (0 failed), at 0.003 hits/s, average response time is 0.001 seconds",
"operationId": "6e78e253-1f15-4ecd-9147-ba407c6fbeac", "operationId": "14d5b1c2-dc03-4ee5-baaa-5c7992acc82e",
"responses": { "responses": {
"200": { "200": {
"description": "Successful call with status 200", "description": "Successful call with status 200",
@ -96,15 +98,15 @@
} }
} }
}, },
"x-last-seen-ts": 1567750581.757122,
"x-counters-total": { "x-counters-total": {
"entries": 3, "entries": 3,
"failures": 0, "failures": 0,
"firstSeen": 1567750581.7471218, "firstSeen": 1567750581.7471218,
"lastSeen": 1567750581.757122, "lastSeen": 1567750581.757122,
"sumRT": 0.003, "sumRT": 0.003,
"sumDuration": 0 "sumDuration": 0.010000228881835938
}, },
"x-sample-entry": 0,
"x-counters-per-source": { "x-counters-per-source": {
"": { "": {
"entries": 3, "entries": 3,
@ -112,9 +114,10 @@
"firstSeen": 1567750581.7471218, "firstSeen": 1567750581.7471218,
"lastSeen": 1567750581.757122, "lastSeen": 1567750581.757122,
"sumRT": 0.003, "sumRT": 0.003,
"sumDuration": 0 "sumDuration": 0.010000228881835938
} }
}, },
"x-last-seen-ts": 1567750581.757122,
"requestBody": { "requestBody": {
"description": "Generic request body", "description": "Generic request body",
"content": { "content": {
@ -129,7 +132,7 @@
"post": { "post": {
"summary": "/body-required", "summary": "/body-required",
"description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.001 seconds", "description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.001 seconds",
"operationId": "1eefebc8-28c0-43d8-877f-4c5f67ff479d", "operationId": "d0958c5a-dce6-4616-99f4-201dbc51457a",
"responses": { "responses": {
"200": { "200": {
"description": "Successful call with status 200", "description": "Successful call with status 200",
@ -138,15 +141,6 @@
} }
} }
}, },
"x-last-seen-ts": 1567750581.757122,
"x-counters-total": {
"entries": 1,
"failures": 0,
"firstSeen": 1567750581.757122,
"lastSeen": 1567750581.757122,
"sumRT": 0.001,
"sumDuration": 0
},
"x-counters-per-source": { "x-counters-per-source": {
"": { "": {
"entries": 1, "entries": 1,
@ -157,6 +151,16 @@
"sumDuration": 0 "sumDuration": 0
} }
}, },
"x-sample-entry": 0,
"x-counters-total": {
"entries": 1,
"failures": 0,
"firstSeen": 1567750581.757122,
"lastSeen": 1567750581.757122,
"sumRT": 0.001,
"sumDuration": 0
},
"x-last-seen-ts": 1567750581.757122,
"requestBody": { "requestBody": {
"description": "Generic request body", "description": "Generic request body",
"content": { "content": {
@ -172,7 +176,7 @@
"post": { "post": {
"summary": "/form-multipart", "summary": "/form-multipart",
"description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.001 seconds", "description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.001 seconds",
"operationId": "fc1eb6a3-3d55-4b4a-bfd0-c2ae85d590d5", "operationId": "cab5a2f3-c18a-4d5a-8f92-e40da4fd6603",
"responses": { "responses": {
"200": { "200": {
"description": "Successful call with status 200", "description": "Successful call with status 200",
@ -183,6 +187,7 @@
} }
} }
}, },
"x-sample-entry": 0,
"x-counters-per-source": { "x-counters-per-source": {
"": { "": {
"entries": 1, "entries": 1,
@ -239,7 +244,7 @@
"post": { "post": {
"summary": "/form-urlencoded", "summary": "/form-urlencoded",
"description": "Mizu observed 2 entries (0 failed), at 0.500 hits/s, average response time is 0.001 seconds", "description": "Mizu observed 2 entries (0 failed), at 0.500 hits/s, average response time is 0.001 seconds",
"operationId": "b416c98b-4ba8-4739-bd2a-6c3e31478bc9", "operationId": "7c373ad7-6ab5-422e-971b-1cf56b18a7a2",
"responses": { "responses": {
"200": { "200": {
"description": "Successful call with status 200", "description": "Successful call with status 200",
@ -248,6 +253,7 @@
} }
} }
}, },
"x-last-seen-ts": 1567750581.7471218,
"x-counters-total": { "x-counters-total": {
"entries": 2, "entries": 2,
"failures": 0, "failures": 0,
@ -266,7 +272,7 @@
"sumDuration": 1 "sumDuration": 1
} }
}, },
"x-last-seen-ts": 1567750581.7471218, "x-sample-entry": 0,
"requestBody": { "requestBody": {
"description": "Generic request body", "description": "Generic request body",
"content": { "content": {
@ -317,7 +323,7 @@
"get": { "get": {
"summary": "/{Id}", "summary": "/{Id}",
"description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.630 seconds", "description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.630 seconds",
"operationId": "d8b4de17-4c0e-401c-891b-36a7cee390bf", "operationId": "99f1d11f-29c0-48f9-8bf0-9f4b407c7c3f",
"responses": { "responses": {
"200": { "200": {
"description": "Successful call with status 200", "description": "Successful call with status 200",
@ -328,6 +334,7 @@
} }
} }
}, },
"x-sample-entry": 0,
"x-last-seen-ts": 1567750579.7471218, "x-last-seen-ts": 1567750579.7471218,
"x-counters-total": { "x-counters-total": {
"entries": 1, "entries": 1,
@ -372,7 +379,7 @@
"get": { "get": {
"summary": "/{Id}/sub1", "summary": "/{Id}/sub1",
"description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.111 seconds", "description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.111 seconds",
"operationId": "edacc275-1761-4011-af9b-3a3ed7c3d2b8", "operationId": "f7e299d2-253c-4eef-975c-9a5659a7fc50",
"responses": { "responses": {
"200": { "200": {
"description": "Successful call with status 200", "description": "Successful call with status 200",
@ -381,6 +388,7 @@
} }
} }
}, },
"x-last-seen-ts": 1567750483.864529,
"x-counters-total": { "x-counters-total": {
"entries": 1, "entries": 1,
"failures": 0, "failures": 0,
@ -399,7 +407,7 @@
"sumDuration": 0 "sumDuration": 0
} }
}, },
"x-last-seen-ts": 1567750483.864529 "x-sample-entry": 0
}, },
"parameters": [ "parameters": [
{ {
@ -425,7 +433,7 @@
"get": { "get": {
"summary": "/{Id}/sub2", "summary": "/{Id}/sub2",
"description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.630 seconds", "description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.630 seconds",
"operationId": "61702da5-e4e7-4d95-8071-a6208f3d19d4", "operationId": "23a54e06-4298-4ea5-b1f0-09b0354a0598",
"responses": { "responses": {
"200": { "200": {
"description": "Successful call with status 200", "description": "Successful call with status 200",
@ -454,7 +462,8 @@
"sumRT": 0.63, "sumRT": 0.63,
"sumDuration": 0 "sumDuration": 0
} }
} },
"x-sample-entry": 0
}, },
"parameters": [ "parameters": [
{ {
@ -477,6 +486,14 @@
] ]
} }
}, },
"x-counters-total": {
"entries": 13,
"failures": 0,
"firstSeen": 1567750483.864529,
"lastSeen": 1567750582.7471218,
"sumRT": 3.268,
"sumDuration": 2.010000228881836
},
"x-counters-per-source": { "x-counters-per-source": {
"": { "": {
"entries": 13, "entries": 13,
@ -484,15 +501,7 @@
"firstSeen": 1567750483.864529, "firstSeen": 1567750483.864529,
"lastSeen": 1567750582.7471218, "lastSeen": 1567750582.7471218,
"sumRT": 3.268, "sumRT": 3.268,
"sumDuration": 2 "sumDuration": 2.010000228881836
} }
},
"x-counters-total": {
"entries": 13,
"failures": 0,
"firstSeen": 1567750483.864529,
"lastSeen": 1567750582.7471218,
"sumRT": 3.268,
"sumDuration": 2
} }
} }

4
agent/pkg/oas/utils.go
View File

@ -115,10 +115,10 @@ type nvParams struct {
GeneralizeName func(name string) string GeneralizeName func(name string) string
} }
func handleNameVals(gw nvParams, params **openapi.ParameterList) { func handleNameVals(gw nvParams, params **openapi.ParameterList, checkIgnore bool) {
visited := map[string]*openapi.ParameterObj{} visited := map[string]*openapi.ParameterObj{}
for _, pair := range gw.Pairs { for _, pair := range gw.Pairs {
if gw.IsIgnored(pair.Name) { if (checkIgnore && gw.IsIgnored(pair.Name)) || pair.Name == "" {
continue continue
} }