diff --git a/agent/pkg/oas/feeder_test.go b/agent/pkg/oas/feeder_test.go index e3be929e8..b3d97d8f0 100644 --- a/agent/pkg/oas/feeder_test.go +++ b/agent/pkg/oas/feeder_test.go @@ -51,32 +51,42 @@ func fileSize(fname string) int64 { return fi.Size() } -func feedEntries(fromFiles []string) (err error) { +func feedEntries(fromFiles []string, isSync bool) (count int, err error) { + badFiles := make([]string, 0) + cnt := 0 for _, file := range fromFiles { logger.Log.Info("Processing file: " + file) ext := strings.ToLower(filepath.Ext(file)) + eCnt := 0 switch ext { case ".har": - err = feedFromHAR(file) + eCnt, err = feedFromHAR(file, isSync) if err != nil { logger.Log.Warning("Failed processing file: " + err.Error()) + badFiles = append(badFiles, file) continue } case ".ldjson": - err = feedFromLDJSON(file) + eCnt, err = feedFromLDJSON(file, isSync) if err != nil { logger.Log.Warning("Failed processing file: " + err.Error()) + badFiles = append(badFiles, file) continue } default: - return errors.New("Unsupported file extension: " + ext) + return 0, errors.New("Unsupported file extension: " + ext) } + cnt += eCnt } - return nil + for _, f := range badFiles { + logger.Log.Infof("Bad file: %s", f) + } + + return cnt, nil } -func feedFromHAR(file string) error { +func feedFromHAR(file string, isSync bool) (int, error) { fd, err := os.Open(file) if err != nil { panic(err) @@ -86,23 +96,43 @@ func feedFromHAR(file string) error { data, err := ioutil.ReadAll(fd) if err != nil { - return err + return 0, err } var harDoc har.HAR err = json.Unmarshal(data, &harDoc) if err != nil { - return err + return 0, err } + cnt := 0 for _, entry := range harDoc.Log.Entries { - GetOasGeneratorInstance().PushEntry(&entry) + cnt += 1 + feedEntry(&entry, isSync) } - return nil + return cnt, nil } -func feedFromLDJSON(file string) error { +func feedEntry(entry *har.Entry, isSync bool) { + if entry.Response.Status == 302 { + logger.Log.Debugf("Dropped traffic entry due to permanent redirect status: %s", entry.StartedDateTime) + } + + if strings.Contains(entry.Request.URL, "taboola") { + logger.Log.Debugf("Interesting: %s", entry.Request.URL) + } else { + //return + } + + if isSync { + GetOasGeneratorInstance().entriesChan <- *entry // blocking variant, right? + } else { + GetOasGeneratorInstance().PushEntry(entry) + } +} + +func feedFromLDJSON(file string, isSync bool) (int, error) { fd, err := os.Open(file) if err != nil { panic(err) @@ -113,8 +143,8 @@ func feedFromLDJSON(file string) error { reader := bufio.NewReader(fd) var meta map[string]interface{} - buf := strings.Builder{} + cnt := 0 for { substr, isPrefix, err := reader.ReadLine() if err == io.EOF { @@ -132,26 +162,28 @@ func feedFromLDJSON(file string) error { if meta == nil { err := json.Unmarshal([]byte(line), &meta) if err != nil { - return err + return 0, err } } else { var entry har.Entry err := json.Unmarshal([]byte(line), &entry) if err != nil { logger.Log.Warningf("Failed decoding entry: %s", line) + } else { + cnt += 1 + feedEntry(&entry, isSync) } - GetOasGeneratorInstance().PushEntry(&entry) } } - return nil + return cnt, nil } func TestFilesList(t *testing.T) { res, err := getFiles("./test_artifacts/") t.Log(len(res)) t.Log(res) - if err != nil || len(res) != 2 { + if err != nil || len(res) != 3 { t.Logf("Should return 2 files but returned %d", len(res)) t.FailNow() } diff --git a/agent/pkg/oas/gibberish.go b/agent/pkg/oas/gibberish.go index 12eb64f1f..1bfb34b17 100644 --- a/agent/pkg/oas/gibberish.go +++ b/agent/pkg/oas/gibberish.go @@ -12,6 +12,7 @@ var ( patEmail = regexp.MustCompile(`^\w+([-+.']\w+)*@\w+([-.]\w+)*\.\w+([-.]\w+)*$`) patHexLower = regexp.MustCompile(`(0x)?[0-9a-f]{6,}`) patHexUpper = regexp.MustCompile(`(0x)?[0-9A-F]{6,}`) + patLongNum = regexp.MustCompile(`\d{6,}`) ) func IsGibberish(str string) bool { @@ -27,7 +28,7 @@ func IsGibberish(str string) bool { return true } - if patHexLower.MatchString(str) || patHexUpper.MatchString(str) { + if patHexLower.MatchString(str) || patHexUpper.MatchString(str) || patLongNum.MatchString(str) { return true } diff --git a/agent/pkg/oas/ignores.go b/agent/pkg/oas/ignores.go index f5da0227f..6a14e664a 100644 --- a/agent/pkg/oas/ignores.go +++ b/agent/pkg/oas/ignores.go @@ -17,17 +17,19 @@ var ignoredHeaders = []string{ "x-att-deviceid", "x-correlation-id", "correlation-id", "x-client-data", "x-http-method-override", "x-real-ip", "x-request-id", "x-request-start", "x-requested-with", "x-uidh", "x-same-domain", "x-content-type-options", "x-frame-options", "x-xss-protection", - "x-wap-profile", "x-scheme", - "newrelic", "x-cloud-trace-context", "sentry-trace", - "expires", "set-cookie", "p3p", "location", "content-security-policy", "content-security-policy-report-only", - "last-modified", "content-language", + "x-wap-profile", "x-scheme", "status", "x-cache", "x-application-context", "retry-after", + "newrelic", "x-cloud-trace-context", "sentry-trace", "x-cache-hits", "x-served-by", "x-span-name", + "expires", "set-cookie", "p3p", "content-security-policy", "content-security-policy-report-only", + "last-modified", "content-language", "x-varnish", "true-client-ip", "akamai-origin-hop", "keep-alive", "etag", "alt-svc", "x-csrf-token", "x-ua-compatible", "vary", "x-powered-by", - "age", "allow", "www-authenticate", + "age", "allow", "www-authenticate", "expect-ct", "timing-allow-origin", "referrer-policy", + "x-aspnet-version", "x-aspnetmvc-version", "x-timer", "x-abuse-info", "x-mod-pagespeed", + "duration_ms", // UP9 custom } var ignoredHeaderPrefixes = []string{ ":", "accept-", "access-control-", "if-", "sec-", "grpc-", - "x-forwarded-", "x-original-", + "x-forwarded-", "x-original-", "cf-", "x-up9-", "x-envoy-", "x-hasura-", "x-b3-", "x-datadog-", "x-envoy-", "x-amz-", "x-newrelic-", "x-prometheus-", "x-akamai-", "x-spotim-", "x-amzn-", "x-ratelimit-", } diff --git a/agent/pkg/oas/specgen.go b/agent/pkg/oas/specgen.go index 3989fd59f..1b17e4d51 100644 --- a/agent/pkg/oas/specgen.go +++ b/agent/pkg/oas/specgen.go @@ -30,7 +30,7 @@ func NewGen(server string) *SpecGen { spec.Version = "3.1.0" info := openapi.Info{Title: server} - info.Version = "0.0" + info.Version = "1.0" spec.Info = &info spec.Paths = &openapi.Paths{Items: map[openapi.PathValue]*openapi.PathObj{}} @@ -175,11 +175,18 @@ func (g *SpecGen) handlePathObj(entry *har.Entry) (string, error) { if isExtIgnored(urlParsed.Path) { logger.Log.Debugf("Dropped traffic entry due to ignored extension: %s", urlParsed.Path) + return "", nil + } + + if entry.Request.Method == "OPTIONS" { + logger.Log.Debugf("Dropped traffic entry due to its method: %s", urlParsed.Path) + return "", nil } ctype := getRespCtype(&entry.Response) if isCtypeIgnored(ctype) { logger.Log.Debugf("Dropped traffic entry due to ignored response ctype: %s", ctype) + return "", nil } if entry.Response.Status < 100 { @@ -192,9 +199,19 @@ func (g *SpecGen) handlePathObj(entry *har.Entry) (string, error) { return "", nil } - split := strings.Split(urlParsed.Path, "/") + if entry.Response.Status == 502 || entry.Response.Status == 503 || entry.Response.Status == 504 { + logger.Log.Debugf("Dropped traffic entry due to temporary server error: %s", entry.StartedDateTime) + return "", nil + } + + var split []string + if urlParsed.RawPath != "" { + split = strings.Split(urlParsed.RawPath, "/") + } else { + split = strings.Split(urlParsed.Path, "/") + } node := g.tree.getOrSet(split, new(openapi.PathObj)) - opObj, err := handleOpObj(entry, node.ops) + opObj, err := handleOpObj(entry, node.pathObj) if opObj != nil { return opObj.OperationID, err @@ -232,20 +249,16 @@ func handleRequest(req *har.Request, opObj *openapi.Operation, isSuccess bool) e // TODO: we don't handle the situation when header/qstr param can be defined on pathObj level. Also the path param defined on opObj qstrGW := nvParams{ - In: openapi.InQuery, - Pairs: func() []NVPair { - return qstrToNVP(req.QueryString) - }, + In: openapi.InQuery, + Pairs: req.QueryString, IsIgnored: func(name string) bool { return false }, GeneralizeName: func(name string) string { return name }, } handleNameVals(qstrGW, &opObj.Parameters) hdrGW := nvParams{ - In: openapi.InHeader, - Pairs: func() []NVPair { - return hdrToNVP(req.Headers) - }, + In: openapi.InHeader, + Pairs: req.Headers, IsIgnored: isHeaderIgnored, GeneralizeName: strings.ToLower, } @@ -348,7 +361,7 @@ func fillContent(reqResp reqResp, respContent openapi.Content, ctype string, err isBinary, _, text = reqResp.Resp.Content.B64Decoded() } - if !isBinary { + if !isBinary && text != "" { var exampleMsg []byte // try treating it as json any, isJSON := anyJSON(text) diff --git a/agent/pkg/oas/specgen_test.go b/agent/pkg/oas/specgen_test.go index 9a55a6332..f88c502ff 100644 --- a/agent/pkg/oas/specgen_test.go +++ b/agent/pkg/oas/specgen_test.go @@ -3,48 +3,56 @@ package oas import ( "encoding/json" "github.com/chanced/openapi" + "github.com/op/go-logging" "github.com/up9inc/mizu/shared/logger" "io/ioutil" "mizuserver/pkg/har" "os" "strings" "testing" + "time" ) // if started via env, write file into subdir -func writeFiles(label string, spec *openapi.OpenAPI) { +func outputSpec(label string, spec *openapi.OpenAPI, t *testing.T) { + content, err := json.MarshalIndent(spec, "", "\t") + if err != nil { + panic(err) + } + if os.Getenv("MIZU_OAS_WRITE_FILES") != "" { path := "./oas-samples" err := os.MkdirAll(path, 0o755) if err != nil { panic(err) } - content, err := json.MarshalIndent(spec, "", "\t") - if err != nil { - panic(err) - } err = ioutil.WriteFile(path+"/"+label+".json", content, 0644) if err != nil { panic(err) } + t.Logf("Written: %s", label) + } else { + t.Logf("%s", string(content)) } } func TestEntries(t *testing.T) { + logger.InitLoggerStderrOnly(logging.INFO) files, err := getFiles("./test_artifacts/") - // files, err = getFiles("/media/bigdisk/UP9") if err != nil { t.Log(err) t.FailNow() } GetOasGeneratorInstance().Start() + loadStartingOAS() - if err := feedEntries(files); err != nil { + cnt, err := feedEntries(files, true) + if err != nil { t.Log(err) t.Fail() } - loadStartingOAS() + waitQueueProcessed() svcs := strings.Builder{} GetOasGeneratorInstance().ServiceSpecs.Range(func(key, val interface{}) bool { @@ -78,33 +86,35 @@ func TestEntries(t *testing.T) { t.FailNow() } - specText, _ := json.MarshalIndent(spec, "", "\t") - t.Logf("%s", string(specText)) + outputSpec(svc, spec, t) err = spec.Validate() if err != nil { t.Log(err) t.FailNow() } - writeFiles(svc, spec) return true }) + logger.Log.Infof("Total entries: %d", cnt) } -func TestFileLDJSON(t *testing.T) { +func TestFileSingle(t *testing.T) { GetOasGeneratorInstance().Start() - file := "test_artifacts/output_rdwtyeoyrj.har.ldjson" - err := feedFromLDJSON(file) + // loadStartingOAS() + file := "test_artifacts/params.har" + files := []string{file} + cnt, err := feedEntries(files, true) if err != nil { logger.Log.Warning("Failed processing file: " + err.Error()) t.Fail() } - loadStartingOAS() + waitQueueProcessed() - GetOasGeneratorInstance().ServiceSpecs.Range(func(_, val interface{}) bool { + GetOasGeneratorInstance().ServiceSpecs.Range(func(key, val interface{}) bool { + svc := key.(string) gen := val.(*SpecGen) spec, err := gen.GetSpec() if err != nil { @@ -112,8 +122,7 @@ func TestFileLDJSON(t *testing.T) { t.FailNow() } - specText, _ := json.MarshalIndent(spec, "", "\t") - t.Logf("%s", string(specText)) + outputSpec(svc, spec, t) err = spec.Validate() if err != nil { @@ -123,6 +132,19 @@ func TestFileLDJSON(t *testing.T) { return true }) + + logger.Log.Infof("Processed entries: %d", cnt) +} + +func waitQueueProcessed() { + for { + time.Sleep(100 * time.Millisecond) + queue := len(GetOasGeneratorInstance().entriesChan) + logger.Log.Infof("Queue: %d", queue) + if queue < 1 { + break + } + } } func loadStartingOAS() { @@ -155,20 +177,29 @@ func loadStartingOAS() { func TestEntriesNegative(t *testing.T) { files := []string{"invalid"} - err := feedEntries(files) + _, err := feedEntries(files, false) if err == nil { t.Logf("Should have failed") t.Fail() } } +func TestEntriesPositive(t *testing.T) { + files := []string{"test_artifacts/params.har"} + _, err := feedEntries(files, false) + if err != nil { + t.Logf("Failed") + t.Fail() + } +} + func TestLoadValidHAR(t *testing.T) { inp := `{"startedDateTime": "2021-02-03T07:48:12.959000+00:00", "time": 1, "request": {"method": "GET", "url": "http://unresolved_target/1.0.0/health", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [], "queryString": [], "headersSize": -1, "bodySize": -1}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [], "content": {"size": 2, "mimeType": "", "text": "OK"}, "redirectURL": "", "headersSize": -1, "bodySize": 2}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 1}}` var entry *har.Entry var err = json.Unmarshal([]byte(inp), &entry) if err != nil { t.Logf("Failed to decode entry: %s", err) - // t.FailNow() demonstrates the problem of library + t.FailNow() // demonstrates the problem of `martian` HAR library } } diff --git a/agent/pkg/oas/test_artifacts/output_ysuwqrdktj.har b/agent/pkg/oas/test_artifacts/example.har similarity index 99% rename from agent/pkg/oas/test_artifacts/output_ysuwqrdktj.har rename to agent/pkg/oas/test_artifacts/example.har index bdfc4a42d..124fb3344 100644 --- a/agent/pkg/oas/test_artifacts/output_ysuwqrdktj.har +++ b/agent/pkg/oas/test_artifacts/example.har @@ -13,10 +13,14 @@ "time": 1022, "request": { "method": "GET", - "url": "http://trcc-api-service/proxies/", + "url": "http://trcc-api-service/proxies;matrixparam=example/", "httpVersion": "", "cookies": [], "headers": [ + { + "name": "X-Custom-Demo-Header", + "value": "demo" + }, { "name": "Sec-Fetch-Dest", "value": "empty" @@ -124,6 +128,10 @@ "httpVersion": "", "cookies": [], "headers": [ + { + "name": "X-Custom-Demo-Header2", + "value": "demo2" + }, { "name": "Vary", "value": "Origin" @@ -24568,7 +24576,7 @@ "bodySize": -1 }, "response": { - "status": 200, + "status": 308, "statusText": "OK", "httpVersion": "", "cookies": [], @@ -24635,7 +24643,7 @@ "time": 1, "request": { "method": "GET", - "url": "http://trcc-api-service/models/aws_kong5/suites/all/runs", + "url": "http://trcc-api-service/models/aws_kong5/suites/all/runs.png", "httpVersion": "", "cookies": [], "headers": [ @@ -24894,7 +24902,7 @@ "bodySize": -1 }, "response": { - "status": 200, + "status": 0, "statusText": "OK", "httpVersion": "", "cookies": [], diff --git a/agent/pkg/oas/test_artifacts/output_rdwtyeoyrj.har.ldjson b/agent/pkg/oas/test_artifacts/example.ldjson similarity index 100% rename from agent/pkg/oas/test_artifacts/output_rdwtyeoyrj.har.ldjson rename to agent/pkg/oas/test_artifacts/example.ldjson diff --git a/agent/pkg/oas/test_artifacts/params.har b/agent/pkg/oas/test_artifacts/params.har new file mode 100644 index 000000000..60a557414 --- /dev/null +++ b/agent/pkg/oas/test_artifacts/params.har @@ -0,0 +1,127 @@ +{ + "log": { + "version": "1.2", + "creator": { + "name": "mitmproxy har_dump", + "version": "0.1", + "comment": "mitmproxy version mitmproxy 4.0.4" + }, + "entries": [ + { + "startedDateTime": "2019-09-06T06:14:43.864529+00:00", + "time": 111, + "request": { + "method": "GET", + "url": "https://httpbin.org/e21f7112-3d3b-4632-9da3-a4af2e0e9166/sub1", + "httpVersion": "HTTP/1.1", + "cookies": [], + "headers": [], + "headersSize": 1542, + "bodySize": 0, + "queryString": [] + }, + "response": { + "status": 200, + "statusText": "OK", + "httpVersion": "HTTP/1.1", + "cookies": [], + "headers": [ + ], + "content": { + "mimeType": "text/html", + "text": "", + "size": 0 + }, + "redirectURL": "", + "headersSize": 245, + "bodySize": 39 + }, + "cache": {}, + "timings": { + "send": 22, + "receive": 2, + "wait": 87, + "connect": -1, + "ssl": -1 + }, + "serverIPAddress": "54.210.29.33" + }, + { + "startedDateTime": "2019-09-06T06:16:18.747122+00:00", + "time": 630, + "request": { + "method": "GET", + "url": "https://httpbin.org/952bea17-3776-11ea-9341-42010a84012a/sub2", + "httpVersion": "HTTP/1.1", + "cookies": [], + "headers": [], + "queryString": [], + "headersSize": 1542, + "bodySize": 0 + }, + "response": { + "status": 200, + "statusText": "OK", + "httpVersion": "HTTP/1.1", + "cookies": [], + "headers": [], + "content": { + "size": 39, + "compression": -20, + "mimeType": "application/json", + "text": "null" + }, + "redirectURL": "", + "headersSize": 248, + "bodySize": 39 + }, + "cache": {}, + "timings": { + "send": 14, + "receive": 4, + "wait": 350, + "connect": 262, + "ssl": -1 + } + }, + { + "startedDateTime": "2019-09-06T06:16:19.747122+00:00", + "time": 630, + "request": { + "method": "GET", + "url": "https://httpbin.org/952bea17-3776-11ea-9341-42010a84012a;mparam=matrixparam", + "httpVersion": "HTTP/1.1", + "cookies": [], + "headers": [], + "queryString": [], + "headersSize": 1542, + "bodySize": 0 + }, + "response": { + "status": 200, + "statusText": "OK", + "httpVersion": "HTTP/1.1", + "cookies": [], + "headers": [], + "content": { + "size": 39, + "compression": -20, + "mimeType": "application/json", + "text": "null" + }, + "redirectURL": "", + "headersSize": 248, + "bodySize": 39 + }, + "cache": {}, + "timings": { + "send": 14, + "receive": 4, + "wait": 350, + "connect": 262, + "ssl": -1 + } + } + ] + } +} \ No newline at end of file diff --git a/agent/pkg/oas/tree.go b/agent/pkg/oas/tree.go index 7aaa87d5e..6824a05c3 100644 --- a/agent/pkg/oas/tree.go +++ b/agent/pkg/oas/tree.go @@ -3,6 +3,7 @@ package oas import ( "github.com/chanced/openapi" "github.com/up9inc/mizu/shared/logger" + "net/url" "strconv" "strings" ) @@ -10,25 +11,37 @@ import ( type NodePath = []string type Node struct { - constant *string - param *openapi.ParameterObj - ops *openapi.PathObj - parent *Node - children []*Node + constant *string + pathParam *openapi.ParameterObj + pathObj *openapi.PathObj + parent *Node + children []*Node } -func (n *Node) getOrSet(path NodePath, pathObjToSet *openapi.PathObj) (node *Node) { - if pathObjToSet == nil { +func (n *Node) getOrSet(path NodePath, existingPathObj *openapi.PathObj) (node *Node) { + if existingPathObj == nil { panic("Invalid function call") } pathChunk := path[0] + potentialMatrix := strings.SplitN(pathChunk, ";", 2) + if len(potentialMatrix) > 1 { + pathChunk = potentialMatrix[0] + logger.Log.Warningf("URI matrix params are not supported: %s", potentialMatrix[1]) + } + chunkIsParam := strings.HasPrefix(pathChunk, "{") && strings.HasSuffix(pathChunk, "}") + pathChunk, err := url.PathUnescape(pathChunk) + if err != nil { + logger.Log.Warningf("URI segment is not correctly encoded: %s", pathChunk) + // any side effects on continuing? + } + chunkIsGibberish := IsGibberish(pathChunk) && !IsVersionString(pathChunk) var paramObj *openapi.ParameterObj - if chunkIsParam && pathObjToSet != nil && pathObjToSet.Parameters != nil { - paramObj = findParamByName(pathObjToSet.Parameters, openapi.InPath, pathChunk[1:len(pathChunk)-1]) + if chunkIsParam && existingPathObj != nil && existingPathObj.Parameters != nil { + _, paramObj = findParamByName(existingPathObj.Parameters, openapi.InPath, pathChunk[1:len(pathChunk)-1]) } if paramObj == nil { @@ -46,34 +59,30 @@ func (n *Node) getOrSet(path NodePath, pathObjToSet *openapi.PathObj) (node *Nod n.children = append(n.children, node) if paramObj != nil { - node.param = paramObj + node.pathParam = paramObj } else if chunkIsGibberish { - initParams(&pathObjToSet.Parameters) newParam := n.createParam() - node.param = newParam - - appended := append(*pathObjToSet.Parameters, newParam) - pathObjToSet.Parameters = &appended + node.pathParam = newParam } else { node.constant = &pathChunk } } - // add example if it's a param - if node.param != nil && !chunkIsParam { - exmp := &node.param.Examples + // add example if it's a gibberish chunk + if node.pathParam != nil && !chunkIsParam { + exmp := &node.pathParam.Examples err := fillParamExample(&exmp, pathChunk) if err != nil { logger.Log.Warningf("Failed to add example to a parameter: %s", err) } } - // TODO: eat up trailing slash, in a smart way: node.ops!=nil && path[1]=="" + // TODO: eat up trailing slash, in a smart way: node.pathObj!=nil && path[1]=="" if len(path) > 1 { - return node.getOrSet(path[1:], pathObjToSet) - } else if node.ops == nil { - node.ops = pathObjToSet + return node.getOrSet(path[1:], existingPathObj) + } else if node.pathObj == nil { + node.pathObj = existingPathObj } return node @@ -90,12 +99,16 @@ func (n *Node) createParam() *openapi.ParameterObj { } else if strings.HasSuffix(*n.constant, "s") && len(*n.constant) > 3 { name = *n.constant name = name[:len(name)-1] + "Id" + } else if isAlpha(*n.constant) { + name = *n.constant + "Id" } + + name = cleanNonAlnum([]byte(name)) } newParam := createSimpleParam(name, "path", "string") x := n.countParentParams() - if x > 1 { + if x > 0 { newParam.Name = newParam.Name + strconv.Itoa(x) } @@ -113,7 +126,7 @@ func (n *Node) searchInParams(paramObj *openapi.ParameterObj, chunkIsGibberish b // TODO: check the regex pattern of param? for exceptions etc if paramObj != nil { - // TODO: mergeParam(subnode.param, paramObj) + // TODO: mergeParam(subnode.pathParam, paramObj) return subnode } else { return subnode @@ -147,15 +160,16 @@ func (n *Node) listPaths() *openapi.Paths { var strChunk string if n.constant != nil { strChunk = *n.constant - } else if n.param != nil { - strChunk = "{" + n.param.Name + "}" + } else if n.pathParam != nil { + strChunk = "{" + n.pathParam.Name + "}" } else { // this is the root node } // add self - if n.ops != nil { - paths.Items[openapi.PathValue(strChunk)] = n.ops + if n.pathObj != nil { + fillPathParams(n, n.pathObj) + paths.Items[openapi.PathValue(strChunk)] = n.pathObj } // recurse into children @@ -175,6 +189,29 @@ func (n *Node) listPaths() *openapi.Paths { return paths } +func fillPathParams(n *Node, pathObj *openapi.PathObj) { + // collect all path parameters from parent hierarchy + node := n + for { + if node.pathParam != nil { + initParams(&pathObj.Parameters) + + idx, paramObj := findParamByName(pathObj.Parameters, openapi.InPath, node.pathParam.Name) + if paramObj == nil { + appended := append(*pathObj.Parameters, node.pathParam) + pathObj.Parameters = &appended + } else { + (*pathObj.Parameters)[idx] = paramObj + } + } + + node = node.parent + if node == nil { + break + } + } +} + type PathAndOp struct { path string op *openapi.Operation @@ -194,7 +231,7 @@ func (n *Node) countParentParams() int { res := 0 node := n for { - if node.param != nil { + if node.pathParam != nil { res++ } diff --git a/agent/pkg/oas/tree_test.go b/agent/pkg/oas/tree_test.go index e6fe33066..fe9415278 100644 --- a/agent/pkg/oas/tree_test.go +++ b/agent/pkg/oas/tree_test.go @@ -8,19 +8,30 @@ import ( func TestTree(t *testing.T) { testCases := []struct { - inp string + inp string + numParams int + label string }{ - {"/"}, - {"/v1.0.0/config/launcher/sp_nKNHCzsN/f34efcae-6583-11eb-908a-00b0fcb9d4f6/vendor,init,conversation"}, + {"/", 0, ""}, + {"/v1.0.0/config/launcher/sp_nKNHCzsN/f34efcae-6583-11eb-908a-00b0fcb9d4f6/vendor,init,conversation", 1, "vendor,init,conversation"}, + {"/v1.0.0/config/launcher/sp_nKNHCzsN/{f34efcae-6583-11eb-908a-00b0fcb9d4f6}/vendor,init,conversation", 0, "vendor,init,conversation"}, + {"/getSvgs/size/small/brand/SFLY/layoutId/170943/layoutVersion/1/sizeId/742/surface/0/isLandscape/true/childSkus/%7B%7D", 1, ""}, } tree := new(Node) for _, tc := range testCases { split := strings.Split(tc.inp, "/") - node := tree.getOrSet(split, new(openapi.PathObj)) + pathObj := new(openapi.PathObj) + node := tree.getOrSet(split, pathObj) + + fillPathParams(node, pathObj) - if node.constant == nil { - t.Errorf("nil constant: %s", tc.inp) + if node.constant != nil && *node.constant != tc.label { + t.Errorf("Constant does not match: %s != %s", *node.constant, tc.label) + } + + if tc.numParams > 0 && (pathObj.Parameters == nil || len(*pathObj.Parameters) < tc.numParams) { + t.Errorf("Wrong num of params, expected: %d", tc.numParams) } } } diff --git a/agent/pkg/oas/utils.go b/agent/pkg/oas/utils.go index 86fb481ed..6979daeb6 100644 --- a/agent/pkg/oas/utils.go +++ b/agent/pkg/oas/utils.go @@ -71,9 +71,10 @@ func createSimpleParam(name string, in openapi.In, ptype openapi.SchemaType) *op return &newParam } -func findParamByName(params *openapi.ParameterList, in openapi.In, name string) (pathParam *openapi.ParameterObj) { +func findParamByName(params *openapi.ParameterList, in openapi.In, name string) (idx int, pathParam *openapi.ParameterObj) { caseInsensitive := in == openapi.InHeader - for _, param := range *params { + for i, param := range *params { + idx = i paramObj, err := param.ResolveParameter(paramResolver) if err != nil { logger.Log.Warningf("Failed to resolve reference: %s", err) @@ -89,7 +90,8 @@ func findParamByName(params *openapi.ParameterList, in openapi.In, name string) break } } - return pathParam + + return idx, pathParam } func findHeaderByName(headers *openapi.Headers, name string) *openapi.HeaderObj { @@ -107,37 +109,16 @@ func findHeaderByName(headers *openapi.Headers, name string) *openapi.HeaderObj return nil } -type NVPair struct { - Name string - Value string -} - type nvParams struct { In openapi.In - Pairs func() []NVPair + Pairs []har.NVP IsIgnored func(name string) bool GeneralizeName func(name string) string } -func qstrToNVP(list []har.QueryString) []NVPair { - res := make([]NVPair, len(list)) - for idx, val := range list { - res[idx] = NVPair{Name: val.Name, Value: val.Value} - } - return res -} - -func hdrToNVP(list []har.Header) []NVPair { - res := make([]NVPair, len(list)) - for idx, val := range list { - res[idx] = NVPair{Name: val.Name, Value: val.Value} - } - return res -} - func handleNameVals(gw nvParams, params **openapi.ParameterList) { visited := map[string]*openapi.ParameterObj{} - for _, pair := range gw.Pairs() { + for _, pair := range gw.Pairs { if gw.IsIgnored(pair.Name) { continue } @@ -145,7 +126,7 @@ func handleNameVals(gw nvParams, params **openapi.ParameterList) { nameGeneral := gw.GeneralizeName(pair.Name) initParams(params) - param := findParamByName(*params, gw.In, pair.Name) + _, param := findParamByName(*params, gw.In, pair.Name) if param == nil { param = createSimpleParam(nameGeneral, gw.In, openapi.TypeString) appended := append(**params, param) @@ -342,3 +323,26 @@ func anyJSON(text string) (anyVal interface{}, isJSON bool) { return nil, false } + +func cleanNonAlnum(s []byte) string { + j := 0 + for _, b := range s { + if ('a' <= b && b <= 'z') || + ('A' <= b && b <= 'Z') || + ('0' <= b && b <= '9') || + b == ' ' { + s[j] = b + j++ + } + } + return string(s[:j]) +} + +func isAlpha(s string) bool { + for _, r := range s { + if (r < 'a' || r > 'z') && (r < 'A' || r > 'Z') { + return false + } + } + return true +}