github/kubeshark
1
0
Fork 0
mirror of https://github.com/kubeshark/kubeshark.git synced 2025-08-02 00:57:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

OAS: series of small improvements (#700)

Browse Source
This commit is contained in:
Andrey Pokhilko 2022-01-27 17:17:21 +03:00 committed by GitHub
parent 7fa1a191a6
commit d011478a74
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 387 additions and 121 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

64
agent/pkg/oas/feeder_test.go
View File

@ -51,32 +51,42 @@ func fileSize(fname string) int64 {
return fi.Size()
}
func feedEntries(fromFiles []string) (err error) {
func feedEntries(fromFiles []string, isSync bool) (count int, err error) {
badFiles := make([]string, 0)
cnt := 0
for _, file := range fromFiles {
logger.Log.Info("Processing file: " + file)
ext := strings.ToLower(filepath.Ext(file))
eCnt := 0
switch ext {
case ".har":
err = feedFromHAR(file)
eCnt, err = feedFromHAR(file, isSync)
if err != nil {
logger.Log.Warning("Failed processing file: " + err.Error())
badFiles = append(badFiles, file)
continue
}
case ".ldjson":
err = feedFromLDJSON(file)
eCnt, err = feedFromLDJSON(file, isSync)
if err != nil {
logger.Log.Warning("Failed processing file: " + err.Error())
badFiles = append(badFiles, file)
continue
}
default:
return errors.New("Unsupported file extension: " + ext)
return 0, errors.New("Unsupported file extension: " + ext)
}
cnt += eCnt
}
return nil
for _, f := range badFiles {
logger.Log.Infof("Bad file: %s", f)
}
return cnt, nil
}
func feedFromHAR(file string) error {
func feedFromHAR(file string, isSync bool) (int, error) {
fd, err := os.Open(file)
if err != nil {
panic(err)
@ -86,23 +96,43 @@ func feedFromHAR(file string) error {
data, err := ioutil.ReadAll(fd)
if err != nil {
return err
return 0, err
}
var harDoc har.HAR
err = json.Unmarshal(data, &harDoc)
if err != nil {
return err
return 0, err
}
cnt := 0
for _, entry := range harDoc.Log.Entries {
GetOasGeneratorInstance().PushEntry(&entry)
cnt += 1
feedEntry(&entry, isSync)
}
return nil
return cnt, nil
}
func feedFromLDJSON(file string) error {
func feedEntry(entry *har.Entry, isSync bool) {
if entry.Response.Status == 302 {
logger.Log.Debugf("Dropped traffic entry due to permanent redirect status: %s", entry.StartedDateTime)
}
if strings.Contains(entry.Request.URL, "taboola") {
logger.Log.Debugf("Interesting: %s", entry.Request.URL)
} else {
//return
}
if isSync {
GetOasGeneratorInstance().entriesChan <- *entry // blocking variant, right?
} else {
GetOasGeneratorInstance().PushEntry(entry)
}
}
func feedFromLDJSON(file string, isSync bool) (int, error) {
fd, err := os.Open(file)
if err != nil {
panic(err)
@ -113,8 +143,8 @@ func feedFromLDJSON(file string) error {
reader := bufio.NewReader(fd)
var meta map[string]interface{}
buf := strings.Builder{}
cnt := 0
for {
substr, isPrefix, err := reader.ReadLine()
if err == io.EOF {
@ -132,26 +162,28 @@ func feedFromLDJSON(file string) error {
if meta == nil {
err := json.Unmarshal([]byte(line), &meta)
if err != nil {
return err
return 0, err
}
} else {
var entry har.Entry
err := json.Unmarshal([]byte(line), &entry)
if err != nil {
logger.Log.Warningf("Failed decoding entry: %s", line)
} else {
cnt += 1
feedEntry(&entry, isSync)
}
GetOasGeneratorInstance().PushEntry(&entry)
}
}
return nil
return cnt, nil
}
func TestFilesList(t *testing.T) {
res, err := getFiles("./test_artifacts/")
t.Log(len(res))
t.Log(res)
if err != nil || len(res) != 2 {
if err != nil || len(res) != 3 {
t.Logf("Should return 2 files but returned %d", len(res))
t.FailNow()
}

3
agent/pkg/oas/gibberish.go
View File

@ -12,6 +12,7 @@ var (
patEmail = regexp.MustCompile(`^\w+([-+.']\w+)*@\w+([-.]\w+)*\.\w+([-.]\w+)*$`)
patHexLower = regexp.MustCompile(`(0x)?[0-9a-f]{6,}`)
patHexUpper = regexp.MustCompile(`(0x)?[0-9A-F]{6,}`)
patLongNum = regexp.MustCompile(`\d{6,}`)
)
func IsGibberish(str string) bool {
@ -27,7 +28,7 @@ func IsGibberish(str string) bool {
return true
}
if patHexLower.MatchString(str) || patHexUpper.MatchString(str) {
if patHexLower.MatchString(str) || patHexUpper.MatchString(str) || patLongNum.MatchString(str) {
return true
}

14
agent/pkg/oas/ignores.go
View File

@ -17,17 +17,19 @@ var ignoredHeaders = []string{
"x-att-deviceid", "x-correlation-id", "correlation-id", "x-client-data",
"x-http-method-override", "x-real-ip", "x-request-id", "x-request-start", "x-requested-with", "x-uidh",
"x-same-domain", "x-content-type-options", "x-frame-options", "x-xss-protection",
"x-wap-profile", "x-scheme",
"newrelic", "x-cloud-trace-context", "sentry-trace",
"expires", "set-cookie", "p3p", "location", "content-security-policy", "content-security-policy-report-only",
"last-modified", "content-language",
"x-wap-profile", "x-scheme", "status", "x-cache", "x-application-context", "retry-after",
"newrelic", "x-cloud-trace-context", "sentry-trace", "x-cache-hits", "x-served-by", "x-span-name",
"expires", "set-cookie", "p3p", "content-security-policy", "content-security-policy-report-only",
"last-modified", "content-language", "x-varnish", "true-client-ip", "akamai-origin-hop",
"keep-alive", "etag", "alt-svc", "x-csrf-token", "x-ua-compatible", "vary", "x-powered-by",
"age", "allow", "www-authenticate",
"age", "allow", "www-authenticate", "expect-ct", "timing-allow-origin", "referrer-policy",
"x-aspnet-version", "x-aspnetmvc-version", "x-timer", "x-abuse-info", "x-mod-pagespeed",
"duration_ms", // UP9 custom
}
var ignoredHeaderPrefixes = []string{
":", "accept-", "access-control-", "if-", "sec-", "grpc-",
"x-forwarded-", "x-original-",
"x-forwarded-", "x-original-", "cf-",
"x-up9-", "x-envoy-", "x-hasura-", "x-b3-", "x-datadog-", "x-envoy-", "x-amz-", "x-newrelic-", "x-prometheus-",
"x-akamai-", "x-spotim-", "x-amzn-", "x-ratelimit-",
}

37
agent/pkg/oas/specgen.go
View File

@ -30,7 +30,7 @@ func NewGen(server string) *SpecGen {
spec.Version = "3.1.0"
info := openapi.Info{Title: server}
info.Version = "0.0"
info.Version = "1.0"
spec.Info = &info
spec.Paths = &openapi.Paths{Items: map[openapi.PathValue]*openapi.PathObj{}}
@ -175,11 +175,18 @@ func (g *SpecGen) handlePathObj(entry *har.Entry) (string, error) {
if isExtIgnored(urlParsed.Path) {
logger.Log.Debugf("Dropped traffic entry due to ignored extension: %s", urlParsed.Path)
return "", nil
}
if entry.Request.Method == "OPTIONS" {
logger.Log.Debugf("Dropped traffic entry due to its method: %s", urlParsed.Path)
return "", nil
}
ctype := getRespCtype(&entry.Response)
if isCtypeIgnored(ctype) {
logger.Log.Debugf("Dropped traffic entry due to ignored response ctype: %s", ctype)
return "", nil
}
if entry.Response.Status < 100 {
@ -192,9 +199,19 @@ func (g *SpecGen) handlePathObj(entry *har.Entry) (string, error) {
return "", nil
}
split := strings.Split(urlParsed.Path, "/")
if entry.Response.Status == 502 || entry.Response.Status == 503 || entry.Response.Status == 504 {
logger.Log.Debugf("Dropped traffic entry due to temporary server error: %s", entry.StartedDateTime)
return "", nil
}
var split []string
if urlParsed.RawPath != "" {
split = strings.Split(urlParsed.RawPath, "/")
} else {
split = strings.Split(urlParsed.Path, "/")
}
node := g.tree.getOrSet(split, new(openapi.PathObj))
opObj, err := handleOpObj(entry, node.ops)
opObj, err := handleOpObj(entry, node.pathObj)
if opObj != nil {
return opObj.OperationID, err
@ -232,20 +249,16 @@ func handleRequest(req *har.Request, opObj *openapi.Operation, isSuccess bool) e
// TODO: we don't handle the situation when header/qstr param can be defined on pathObj level. Also the path param defined on opObj
qstrGW := nvParams{
In: openapi.InQuery,
Pairs: func() []NVPair {
return qstrToNVP(req.QueryString)
},
In: openapi.InQuery,
Pairs: req.QueryString,
IsIgnored: func(name string) bool { return false },
GeneralizeName: func(name string) string { return name },
}
handleNameVals(qstrGW, &opObj.Parameters)
hdrGW := nvParams{
In: openapi.InHeader,
Pairs: func() []NVPair {
return hdrToNVP(req.Headers)
},
In: openapi.InHeader,
Pairs: req.Headers,
IsIgnored: isHeaderIgnored,
GeneralizeName: strings.ToLower,
}
@ -348,7 +361,7 @@ func fillContent(reqResp reqResp, respContent openapi.Content, ctype string, err
isBinary, _, text = reqResp.Resp.Content.B64Decoded()
}
if !isBinary {
if !isBinary && text != "" {
var exampleMsg []byte
// try treating it as json
any, isJSON := anyJSON(text)

71
agent/pkg/oas/specgen_test.go
View File

@ -3,48 +3,56 @@ package oas
import (
"encoding/json"
"github.com/chanced/openapi"
"github.com/op/go-logging"
"github.com/up9inc/mizu/shared/logger"
"io/ioutil"
"mizuserver/pkg/har"
"os"
"strings"
"testing"
"time"
)
// if started via env, write file into subdir
func writeFiles(label string, spec *openapi.OpenAPI) {
func outputSpec(label string, spec *openapi.OpenAPI, t *testing.T) {
content, err := json.MarshalIndent(spec, "", "\t")
if err != nil {
panic(err)
}
if os.Getenv("MIZU_OAS_WRITE_FILES") != "" {
path := "./oas-samples"
err := os.MkdirAll(path, 0o755)
if err != nil {
panic(err)
}
content, err := json.MarshalIndent(spec, "", "\t")
if err != nil {
panic(err)
}
err = ioutil.WriteFile(path+"/"+label+".json", content, 0644)
if err != nil {
panic(err)
}
t.Logf("Written: %s", label)
} else {
t.Logf("%s", string(content))
}
}
func TestEntries(t *testing.T) {
logger.InitLoggerStderrOnly(logging.INFO)
files, err := getFiles("./test_artifacts/")
// files, err = getFiles("/media/bigdisk/UP9")
if err != nil {
t.Log(err)
t.FailNow()
}
GetOasGeneratorInstance().Start()
loadStartingOAS()
if err := feedEntries(files); err != nil {
cnt, err := feedEntries(files, true)
if err != nil {
t.Log(err)
t.Fail()
}
loadStartingOAS()
waitQueueProcessed()
svcs := strings.Builder{}
GetOasGeneratorInstance().ServiceSpecs.Range(func(key, val interface{}) bool {
@ -78,33 +86,35 @@ func TestEntries(t *testing.T) {
t.FailNow()
}
specText, _ := json.MarshalIndent(spec, "", "\t")
t.Logf("%s", string(specText))
outputSpec(svc, spec, t)
err = spec.Validate()
if err != nil {
t.Log(err)
t.FailNow()
}
writeFiles(svc, spec)
return true
})
logger.Log.Infof("Total entries: %d", cnt)
}
func TestFileLDJSON(t *testing.T) {
func TestFileSingle(t *testing.T) {
GetOasGeneratorInstance().Start()
file := "test_artifacts/output_rdwtyeoyrj.har.ldjson"
err := feedFromLDJSON(file)
// loadStartingOAS()
file := "test_artifacts/params.har"
files := []string{file}
cnt, err := feedEntries(files, true)
if err != nil {
logger.Log.Warning("Failed processing file: " + err.Error())
t.Fail()
}
loadStartingOAS()
waitQueueProcessed()
GetOasGeneratorInstance().ServiceSpecs.Range(func(_, val interface{}) bool {
GetOasGeneratorInstance().ServiceSpecs.Range(func(key, val interface{}) bool {
svc := key.(string)
gen := val.(*SpecGen)
spec, err := gen.GetSpec()
if err != nil {
@ -112,8 +122,7 @@ func TestFileLDJSON(t *testing.T) {
t.FailNow()
}
specText, _ := json.MarshalIndent(spec, "", "\t")
t.Logf("%s", string(specText))
outputSpec(svc, spec, t)
err = spec.Validate()
if err != nil {
@ -123,6 +132,19 @@ func TestFileLDJSON(t *testing.T) {
return true
})
logger.Log.Infof("Processed entries: %d", cnt)
}
func waitQueueProcessed() {
for {
time.Sleep(100 * time.Millisecond)
queue := len(GetOasGeneratorInstance().entriesChan)
logger.Log.Infof("Queue: %d", queue)
if queue < 1 {
break
}
}
}
func loadStartingOAS() {
@ -155,20 +177,29 @@ func loadStartingOAS() {
func TestEntriesNegative(t *testing.T) {
files := []string{"invalid"}
err := feedEntries(files)
_, err := feedEntries(files, false)
if err == nil {
t.Logf("Should have failed")
t.Fail()
}
}
func TestEntriesPositive(t *testing.T) {
files := []string{"test_artifacts/params.har"}
_, err := feedEntries(files, false)
if err != nil {
t.Logf("Failed")
t.Fail()
}
}
func TestLoadValidHAR(t *testing.T) {
inp := `{"startedDateTime": "2021-02-03T07:48:12.959000+00:00", "time": 1, "request": {"method": "GET", "url": "http://unresolved_target/1.0.0/health", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [], "queryString": [], "headersSize": -1, "bodySize": -1}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [], "content": {"size": 2, "mimeType": "", "text": "OK"}, "redirectURL": "", "headersSize": -1, "bodySize": 2}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 1}}`
var entry *har.Entry
var err = json.Unmarshal([]byte(inp), &entry)
if err != nil {
t.Logf("Failed to decode entry: %s", err)
// t.FailNow() demonstrates the problem of library
t.FailNow() // demonstrates the problem of `martian` HAR library
}
}

16
agent/pkg/oas/test_artifacts/output_ysuwqrdktj.har → agent/pkg/oas/test_artifacts/example.har
View File

@ -13,10 +13,14 @@
"time": 1022,
"request": {
"method": "GET",
"url": "http://trcc-api-service/proxies/",
"url": "http://trcc-api-service/proxies;matrixparam=example/",
"httpVersion": "",
"cookies": [],
"headers": [
{
"name": "X-Custom-Demo-Header",
"value": "demo"
},
{
"name": "Sec-Fetch-Dest",
"value": "empty"
@ -124,6 +128,10 @@
"httpVersion": "",
"cookies": [],
"headers": [
{
"name": "X-Custom-Demo-Header2",
"value": "demo2"
},
{
"name": "Vary",
"value": "Origin"
@ -24568,7 +24576,7 @@
"bodySize": -1
},
"response": {
"status": 200,
"status": 308,
"statusText": "OK",
"httpVersion": "",
"cookies": [],
@ -24635,7 +24643,7 @@
"time": 1,
"request": {
"method": "GET",
"url": "http://trcc-api-service/models/aws_kong5/suites/all/runs",
"url": "http://trcc-api-service/models/aws_kong5/suites/all/runs.png",
"httpVersion": "",
"cookies": [],
"headers": [
@ -24894,7 +24902,7 @@
"bodySize": -1
},
"response": {
"status": 200,
"status": 0,
"statusText": "OK",
"httpVersion": "",
"cookies": [],

0
agent/pkg/oas/test_artifacts/output_rdwtyeoyrj.har.ldjson → agent/pkg/oas/test_artifacts/example.ldjson
View File

127
agent/pkg/oas/test_artifacts/params.har Normal file
View File

@ -0,0 +1,127 @@
{
"log": {
"version": "1.2",
"creator": {
"name": "mitmproxy har_dump",
"version": "0.1",
"comment": "mitmproxy version mitmproxy 4.0.4"
},
"entries": [
{
"startedDateTime": "2019-09-06T06:14:43.864529+00:00",
"time": 111,
"request": {
"method": "GET",
"url": "https://httpbin.org/e21f7112-3d3b-4632-9da3-a4af2e0e9166/sub1",
"httpVersion": "HTTP/1.1",
"cookies": [],
"headers": [],
"headersSize": 1542,
"bodySize": 0,
"queryString": []
},
"response": {
"status": 200,
"statusText": "OK",
"httpVersion": "HTTP/1.1",
"cookies": [],
"headers": [
],
"content": {
"mimeType": "text/html",
"text": "",
"size": 0
},
"redirectURL": "",
"headersSize": 245,
"bodySize": 39
},
"cache": {},
"timings": {
"send": 22,
"receive": 2,
"wait": 87,
"connect": -1,
"ssl": -1
},
"serverIPAddress": "54.210.29.33"
},
{
"startedDateTime": "2019-09-06T06:16:18.747122+00:00",
"time": 630,
"request": {
"method": "GET",
"url": "https://httpbin.org/952bea17-3776-11ea-9341-42010a84012a/sub2",
"httpVersion": "HTTP/1.1",
"cookies": [],
"headers": [],
"queryString": [],
"headersSize": 1542,
"bodySize": 0
},
"response": {
"status": 200,
"statusText": "OK",
"httpVersion": "HTTP/1.1",
"cookies": [],
"headers": [],
"content": {
"size": 39,
"compression": -20,
"mimeType": "application/json",
"text": "null"
},
"redirectURL": "",
"headersSize": 248,
"bodySize": 39
},
"cache": {},
"timings": {
"send": 14,
"receive": 4,
"wait": 350,
"connect": 262,
"ssl": -1
}
},
{
"startedDateTime": "2019-09-06T06:16:19.747122+00:00",
"time": 630,
"request": {
"method": "GET",
"url": "https://httpbin.org/952bea17-3776-11ea-9341-42010a84012a;mparam=matrixparam",
"httpVersion": "HTTP/1.1",
"cookies": [],
"headers": [],
"queryString": [],
"headersSize": 1542,
"bodySize": 0
},
"response": {
"status": 200,
"statusText": "OK",
"httpVersion": "HTTP/1.1",
"cookies": [],
"headers": [],
"content": {
"size": 39,
"compression": -20,
"mimeType": "application/json",
"text": "null"
},
"redirectURL": "",
"headersSize": 248,
"bodySize": 39
},
"cache": {},
"timings": {
"send": 14,
"receive": 4,
"wait": 350,
"connect": 262,
"ssl": -1
}
}
]
}
}

95
agent/pkg/oas/tree.go
View File

@ -3,6 +3,7 @@ package oas
import (
"github.com/chanced/openapi"
"github.com/up9inc/mizu/shared/logger"
"net/url"
"strconv"
"strings"
)
@ -10,25 +11,37 @@ import (
type NodePath = []string
type Node struct {
constant *string
param *openapi.ParameterObj
ops *openapi.PathObj
parent *Node
children []*Node
constant *string
pathParam *openapi.ParameterObj
pathObj *openapi.PathObj
parent *Node
children []*Node
}
func (n *Node) getOrSet(path NodePath, pathObjToSet *openapi.PathObj) (node *Node) {
if pathObjToSet == nil {
func (n *Node) getOrSet(path NodePath, existingPathObj *openapi.PathObj) (node *Node) {
if existingPathObj == nil {
panic("Invalid function call")
}
pathChunk := path[0]
potentialMatrix := strings.SplitN(pathChunk, ";", 2)
if len(potentialMatrix) > 1 {
pathChunk = potentialMatrix[0]
logger.Log.Warningf("URI matrix params are not supported: %s", potentialMatrix[1])
}
chunkIsParam := strings.HasPrefix(pathChunk, "{") && strings.HasSuffix(pathChunk, "}")
pathChunk, err := url.PathUnescape(pathChunk)
if err != nil {
logger.Log.Warningf("URI segment is not correctly encoded: %s", pathChunk)
// any side effects on continuing?
}
chunkIsGibberish := IsGibberish(pathChunk) && !IsVersionString(pathChunk)
var paramObj *openapi.ParameterObj
if chunkIsParam && pathObjToSet != nil && pathObjToSet.Parameters != nil {
paramObj = findParamByName(pathObjToSet.Parameters, openapi.InPath, pathChunk[1:len(pathChunk)-1])
if chunkIsParam && existingPathObj != nil && existingPathObj.Parameters != nil {
_, paramObj = findParamByName(existingPathObj.Parameters, openapi.InPath, pathChunk[1:len(pathChunk)-1])
}
if paramObj == nil {
@ -46,34 +59,30 @@ func (n *Node) getOrSet(path NodePath, pathObjToSet *openapi.PathObj) (node *Nod
n.children = append(n.children, node)
if paramObj != nil {
node.param = paramObj
node.pathParam = paramObj
} else if chunkIsGibberish {
initParams(&pathObjToSet.Parameters)
newParam := n.createParam()
node.param = newParam
appended := append(*pathObjToSet.Parameters, newParam)
pathObjToSet.Parameters = &appended
node.pathParam = newParam
} else {
node.constant = &pathChunk
}
}
// add example if it's a param
if node.param != nil && !chunkIsParam {
exmp := &node.param.Examples
// add example if it's a gibberish chunk
if node.pathParam != nil && !chunkIsParam {
exmp := &node.pathParam.Examples
err := fillParamExample(&exmp, pathChunk)
if err != nil {
logger.Log.Warningf("Failed to add example to a parameter: %s", err)
}
}
// TODO: eat up trailing slash, in a smart way: node.ops!=nil && path[1]==""
// TODO: eat up trailing slash, in a smart way: node.pathObj!=nil && path[1]==""
if len(path) > 1 {
return node.getOrSet(path[1:], pathObjToSet)
} else if node.ops == nil {
node.ops = pathObjToSet
return node.getOrSet(path[1:], existingPathObj)
} else if node.pathObj == nil {
node.pathObj = existingPathObj
}
return node
@ -90,12 +99,16 @@ func (n *Node) createParam() *openapi.ParameterObj {
} else if strings.HasSuffix(*n.constant, "s") && len(*n.constant) > 3 {
name = *n.constant
name = name[:len(name)-1] + "Id"
} else if isAlpha(*n.constant) {
name = *n.constant + "Id"
}
name = cleanNonAlnum([]byte(name))
}
newParam := createSimpleParam(name, "path", "string")
x := n.countParentParams()
if x > 1 {
if x > 0 {
newParam.Name = newParam.Name + strconv.Itoa(x)
}
@ -113,7 +126,7 @@ func (n *Node) searchInParams(paramObj *openapi.ParameterObj, chunkIsGibberish b
// TODO: check the regex pattern of param? for exceptions etc
if paramObj != nil {
// TODO: mergeParam(subnode.param, paramObj)
// TODO: mergeParam(subnode.pathParam, paramObj)
return subnode
} else {
return subnode
@ -147,15 +160,16 @@ func (n *Node) listPaths() *openapi.Paths {
var strChunk string
if n.constant != nil {
strChunk = *n.constant
} else if n.param != nil {
strChunk = "{" + n.param.Name + "}"
} else if n.pathParam != nil {
strChunk = "{" + n.pathParam.Name + "}"
} else {
// this is the root node
}
// add self
if n.ops != nil {
paths.Items[openapi.PathValue(strChunk)] = n.ops
if n.pathObj != nil {
fillPathParams(n, n.pathObj)
paths.Items[openapi.PathValue(strChunk)] = n.pathObj
}
// recurse into children
@ -175,6 +189,29 @@ func (n *Node) listPaths() *openapi.Paths {
return paths
}
func fillPathParams(n *Node, pathObj *openapi.PathObj) {
// collect all path parameters from parent hierarchy
node := n
for {
if node.pathParam != nil {
initParams(&pathObj.Parameters)
idx, paramObj := findParamByName(pathObj.Parameters, openapi.InPath, node.pathParam.Name)
if paramObj == nil {
appended := append(*pathObj.Parameters, node.pathParam)
pathObj.Parameters = &appended
} else {
(*pathObj.Parameters)[idx] = paramObj
}
}
node = node.parent
if node == nil {
break
}
}
}
type PathAndOp struct {
path string
op *openapi.Operation
@ -194,7 +231,7 @@ func (n *Node) countParentParams() int {
res := 0
node := n
for {
if node.param != nil {
if node.pathParam != nil {
res++
}

23
agent/pkg/oas/tree_test.go
View File

@ -8,19 +8,30 @@ import (
func TestTree(t *testing.T) {
testCases := []struct {
inp string
inp string
numParams int
label string
}{
{"/"},
{"/v1.0.0/config/launcher/sp_nKNHCzsN/f34efcae-6583-11eb-908a-00b0fcb9d4f6/vendor,init,conversation"},
{"/", 0, ""},
{"/v1.0.0/config/launcher/sp_nKNHCzsN/f34efcae-6583-11eb-908a-00b0fcb9d4f6/vendor,init,conversation", 1, "vendor,init,conversation"},
{"/v1.0.0/config/launcher/sp_nKNHCzsN/{f34efcae-6583-11eb-908a-00b0fcb9d4f6}/vendor,init,conversation", 0, "vendor,init,conversation"},
{"/getSvgs/size/small/brand/SFLY/layoutId/170943/layoutVersion/1/sizeId/742/surface/0/isLandscape/true/childSkus/%7B%7D", 1, ""},
}
tree := new(Node)
for _, tc := range testCases {
split := strings.Split(tc.inp, "/")
node := tree.getOrSet(split, new(openapi.PathObj))
pathObj := new(openapi.PathObj)
node := tree.getOrSet(split, pathObj)
fillPathParams(node, pathObj)
if node.constant == nil {
t.Errorf("nil constant: %s", tc.inp)
if node.constant != nil && *node.constant != tc.label {
t.Errorf("Constant does not match: %s != %s", *node.constant, tc.label)
}
if tc.numParams > 0 && (pathObj.Parameters == nil || len(*pathObj.Parameters) < tc.numParams) {
t.Errorf("Wrong num of params, expected: %d", tc.numParams)
}
}
}

58
agent/pkg/oas/utils.go
View File

@ -71,9 +71,10 @@ func createSimpleParam(name string, in openapi.In, ptype openapi.SchemaType) *op
return &newParam
}
func findParamByName(params *openapi.ParameterList, in openapi.In, name string) (pathParam *openapi.ParameterObj) {
func findParamByName(params *openapi.ParameterList, in openapi.In, name string) (idx int, pathParam *openapi.ParameterObj) {
caseInsensitive := in == openapi.InHeader
for _, param := range *params {
for i, param := range *params {
idx = i
paramObj, err := param.ResolveParameter(paramResolver)
if err != nil {
logger.Log.Warningf("Failed to resolve reference: %s", err)
@ -89,7 +90,8 @@ func findParamByName(params *openapi.ParameterList, in openapi.In, name string)
break
}
}
return pathParam
return idx, pathParam
}
func findHeaderByName(headers *openapi.Headers, name string) *openapi.HeaderObj {
@ -107,37 +109,16 @@ func findHeaderByName(headers *openapi.Headers, name string) *openapi.HeaderObj
return nil
}
type NVPair struct {
Name string
Value string
}
type nvParams struct {
In openapi.In
Pairs func() []NVPair
Pairs []har.NVP
IsIgnored func(name string) bool
GeneralizeName func(name string) string
}
func qstrToNVP(list []har.QueryString) []NVPair {
res := make([]NVPair, len(list))
for idx, val := range list {
res[idx] = NVPair{Name: val.Name, Value: val.Value}
}
return res
}
func hdrToNVP(list []har.Header) []NVPair {
res := make([]NVPair, len(list))
for idx, val := range list {
res[idx] = NVPair{Name: val.Name, Value: val.Value}
}
return res
}
func handleNameVals(gw nvParams, params **openapi.ParameterList) {
visited := map[string]*openapi.ParameterObj{}
for _, pair := range gw.Pairs() {
for _, pair := range gw.Pairs {
if gw.IsIgnored(pair.Name) {
continue
}
@ -145,7 +126,7 @@ func handleNameVals(gw nvParams, params **openapi.ParameterList) {
nameGeneral := gw.GeneralizeName(pair.Name)
initParams(params)
param := findParamByName(*params, gw.In, pair.Name)
_, param := findParamByName(*params, gw.In, pair.Name)
if param == nil {
param = createSimpleParam(nameGeneral, gw.In, openapi.TypeString)
appended := append(**params, param)
@ -342,3 +323,26 @@ func anyJSON(text string) (anyVal interface{}, isJSON bool) {
return nil, false
}
func cleanNonAlnum(s []byte) string {
j := 0
for _, b := range s {
if ('a' <= b && b <= 'z') ||
('A' <= b && b <= 'Z') ||
('0' <= b && b <= '9') ||
b == ' ' {
s[j] = b
j++
}
}
return string(s[:j])
}
func isAlpha(s string) bool {
for _, r := range s {
if (r < 'a' || r > 'z') && (r < 'A' || r > 'Z') {
return false
}
}
return true
}