From 5934be4da607f608fcda4c624cce79a22584851b Mon Sep 17 00:00:00 2001 From: Andrey Pokhilko Date: Thu, 3 Feb 2022 11:46:31 +0300 Subject: [PATCH] OAS: handle urlencoded and multipart form data correctly (#730) * Start with tests * Expected file * Almost works * Make test stab;e * 5 examples, not 6 * Add param type * multipart example * parsing multipart * Commit * Commit * multipart * Write encoding into schema * Stable test * Add reset * Refactoring * Refactoring * Maintain the required flag * lint --- agent/go.mod | 3 +- agent/go.sum | 4 +- agent/pkg/oas/counters.go | 60 ++- agent/pkg/oas/feeder_test.go | 7 +- agent/pkg/oas/oas_generator.go | 4 + agent/pkg/oas/specgen.go | 195 +++++---- agent/pkg/oas/specgen_test.go | 37 +- agent/pkg/oas/test_artifacts/params.har | 135 ++++++ .../oas/test_artifacts/params.har.spec.json | 413 ++++++++++++++++++ agent/pkg/oas/utils.go | 63 ++- 10 files changed, 833 insertions(+), 88 deletions(-) create mode 100644 agent/pkg/oas/test_artifacts/params.har.spec.json diff --git a/agent/go.mod b/agent/go.mod index 8296d508b..50d296caf 100644 --- a/agent/go.mod +++ b/agent/go.mod @@ -4,7 +4,7 @@ go 1.16 require ( github.com/antelman107/net-wait-go v0.0.0-20210623112055-cf684aebda7b - github.com/chanced/openapi v0.0.6 + github.com/chanced/openapi v0.0.7 github.com/djherbis/atime v1.0.0 github.com/elastic/go-elasticsearch/v7 v7.16.0 github.com/getkin/kin-openapi v0.76.0 @@ -30,6 +30,7 @@ require ( github.com/up9inc/mizu/tap/extensions/http v0.0.0 github.com/up9inc/mizu/tap/extensions/kafka v0.0.0 github.com/up9inc/mizu/tap/extensions/redis v0.0.0 + github.com/wI2L/jsondiff v0.1.1 github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0 k8s.io/api v0.21.2 k8s.io/apimachinery v0.21.2 diff --git a/agent/go.sum b/agent/go.sum index 4b2f13c91..244ad31fc 100644 --- a/agent/go.sum +++ b/agent/go.sum @@ -95,8 +95,8 @@ github.com/chanced/cmpjson v0.0.0-20210415035445-da9262c1f20a h1:zG6t+4krPXcCKtL github.com/chanced/cmpjson v0.0.0-20210415035445-da9262c1f20a/go.mod h1:yhcmlFk1hxuZ+5XZbupzT/cEm/eE4ZvWbmsW1+Q/aZE= github.com/chanced/dynamic v0.0.0-20210502140838-c010b5fc3e44 h1:4NOJMtvZaOA6cI2gkIuXk/2b5KTOvm/R4zyPy/yLCM4= github.com/chanced/dynamic v0.0.0-20210502140838-c010b5fc3e44/go.mod h1:XVNfXN5kgZST4PQ0W/oBAHJku2OteCeHxjAbvfd0ARM= -github.com/chanced/openapi v0.0.6 h1:2giGS47+T8/7MN2hfRHSIUPx86Qksk+J/ciIWcAM7hY= -github.com/chanced/openapi v0.0.6/go.mod h1:SxE2VMLPw+T7Vq8nwbVVhDF2PigvRF4n5XyqsVpRJGU= +github.com/chanced/openapi v0.0.7 h1:OmOBHCg/5ViUg0gaGxXBeEFoVBE8C2pHK4BO/AiD6k8= +github.com/chanced/openapi v0.0.7/go.mod h1:SxE2VMLPw+T7Vq8nwbVVhDF2PigvRF4n5XyqsVpRJGU= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= diff --git a/agent/pkg/oas/counters.go b/agent/pkg/oas/counters.go index cd3b1b328..ff7ac52d4 100644 --- a/agent/pkg/oas/counters.go +++ b/agent/pkg/oas/counters.go @@ -1,6 +1,11 @@ package oas -import "math" +import ( + "fmt" + "github.com/chanced/openapi" + "math" + "strings" +) type Counter struct { Entries int `json:"entries"` @@ -55,3 +60,56 @@ func (m *CounterMap) addOther(other *CounterMap) { } } } + +func setCounterMsgIfOk(oldStr string, cnt *Counter) string { + tpl := "Mizu observed %d entries (%d failed), at %.3f hits/s, average response time is %.3f seconds" + if oldStr == "" || (strings.HasPrefix(oldStr, "Mizu ") && strings.HasSuffix(oldStr, " seconds")) { + return fmt.Sprintf(tpl, cnt.Entries, cnt.Failures, cnt.SumDuration/float64(cnt.Entries), cnt.SumRT/float64(cnt.Entries)) + } + return oldStr +} + +type CounterMaps struct { + counterTotal Counter + counterMapTotal CounterMap +} + +func (m *CounterMaps) processOp(opObj *openapi.Operation) error { + if _, ok := opObj.Extensions.Extension(CountersTotal); ok { + counter := new(Counter) + err := opObj.Extensions.DecodeExtension(CountersTotal, counter) + if err != nil { + return err + } + m.counterTotal.addOther(counter) + + opObj.Description = setCounterMsgIfOk(opObj.Description, counter) + } + + if _, ok := opObj.Extensions.Extension(CountersPerSource); ok { + counterMap := new(CounterMap) + err := opObj.Extensions.DecodeExtension(CountersPerSource, counterMap) + if err != nil { + return err + } + m.counterMapTotal.addOther(counterMap) + } + return nil +} + +func (m *CounterMaps) processOas(oas *openapi.OpenAPI) error { + if oas.Extensions == nil { + oas.Extensions = openapi.Extensions{} + } + + err := oas.Extensions.SetExtension(CountersTotal, m.counterTotal) + if err != nil { + return err + } + + err = oas.Extensions.SetExtension(CountersPerSource, m.counterMapTotal) + if err != nil { + return nil + } + return nil +} diff --git a/agent/pkg/oas/feeder_test.go b/agent/pkg/oas/feeder_test.go index c5fb0c753..59fd451e4 100644 --- a/agent/pkg/oas/feeder_test.go +++ b/agent/pkg/oas/feeder_test.go @@ -110,13 +110,14 @@ func feedFromHAR(file string, isSync bool) (int, error) { cnt := 0 for _, entry := range harDoc.Log.Entries { cnt += 1 - feedEntry(&entry, "", isSync) + feedEntry(&entry, "", isSync, file) } return cnt, nil } -func feedEntry(entry *har.Entry, source string, isSync bool) { +func feedEntry(entry *har.Entry, source string, isSync bool, file string) { + entry.Comment = file if entry.Response.Status == 302 { logger.Log.Debugf("Dropped traffic entry due to permanent redirect status: %s", entry.StartedDateTime) } @@ -176,7 +177,7 @@ func feedFromLDJSON(file string, isSync bool) (int, error) { logger.Log.Warningf("Failed decoding entry: %s", line) } else { cnt += 1 - feedEntry(&entry, source, isSync) + feedEntry(&entry, source, isSync, file) } } } diff --git a/agent/pkg/oas/oas_generator.go b/agent/pkg/oas/oas_generator.go index 27cb01671..7f3aef7ed 100644 --- a/agent/pkg/oas/oas_generator.go +++ b/agent/pkg/oas/oas_generator.go @@ -79,6 +79,10 @@ func (g *oasGenerator) runGeneretor() { } } +func (g *oasGenerator) Reset() { + g.ServiceSpecs = &sync.Map{} +} + func (g *oasGenerator) PushEntry(entryWithSource *EntryWithSource) { if !g.started { return diff --git a/agent/pkg/oas/specgen.go b/agent/pkg/oas/specgen.go index 549851038..a63e3aaaf 100644 --- a/agent/pkg/oas/specgen.go +++ b/agent/pkg/oas/specgen.go @@ -3,13 +3,17 @@ package oas import ( "encoding/json" "errors" - "fmt" "github.com/chanced/openapi" "github.com/google/uuid" "github.com/nav-inc/datetime" "github.com/up9inc/mizu/shared/logger" + "io" + "io/ioutil" "mime" + "mime/multipart" + "net/textproto" "net/url" + "sort" "strconv" "strings" "sync" @@ -126,6 +130,7 @@ func (g *SpecGen) GetSpec() (*openapi.OpenAPI, error) { func suggestTags(oas *openapi.OpenAPI) { paths := getPathsKeys(oas.Paths.Items) + sort.Strings(paths) // make it stable in case of multiple candidates for len(paths) > 0 { group := make([]string, 0) group = append(group, paths[0]) @@ -155,21 +160,9 @@ func suggestTags(oas *openapi.OpenAPI) { } } } - - //groups[common] = group } } -func deleteFromSlice(s []string, val string) []string { - temp := s[:0] - for _, x := range s { - if x != val { - temp = append(temp, x) - } - } - return temp -} - func getPathsKeys(mymap map[openapi.PathValue]*openapi.PathObj) []string { keys := make([]string, len(mymap)) @@ -364,7 +357,7 @@ func handleRequest(req *har.Request, opObj *openapi.Operation, isSuccess bool) e } if reqBody != nil { - reqCtype := getReqCtype(req) + reqCtype, _ := getReqCtype(req) reqMedia, err := fillContent(reqResp{Req: req}, reqBody.Content, reqCtype) if err != nil { return err @@ -471,12 +464,121 @@ func fillContent(reqResp reqResp, respContent openapi.Content, ctype string) (*o } } - content.Example = exampleMsg + if ctype == "application/x-www-form-urlencoded" && reqResp.Req != nil { + handleFormDataUrlencoded(text, content) + } else if strings.HasPrefix(ctype, "multipart/form-data") && reqResp.Req != nil { + _, params := getReqCtype(reqResp.Req) + handleFormDataMultipart(text, content, params) + } + + if content.Example == nil && len(exampleMsg) > len(content.Example) { + content.Example = exampleMsg + } } return respContent[ctype], nil } +func handleFormDataUrlencoded(text string, content *openapi.MediaType) { + formData, err := url.ParseQuery(text) + if err != nil { + logger.Log.Warningf("Could not decode urlencoded: %s", err) + return + } + + parts := make([]PartWithBody, 0) + for name, vals := range formData { + for _, val := range vals { + part := new(multipart.Part) + part.Header = textproto.MIMEHeader{} + part.Header.Add("Content-Disposition", "form-data; name=\""+name+"\";") + parts = append(parts, PartWithBody{part: part, body: []byte(val)}) + } + } + handleFormData(content, parts) +} + +func handleFormData(content *openapi.MediaType, parts []PartWithBody) { + hadSchema := true + if content.Schema == nil { + hadSchema = false // will use it for required flags + content.Schema = new(openapi.SchemaObj) + content.Schema.Type = openapi.Types{openapi.TypeObject} + content.Schema.Properties = openapi.Schemas{} + } + + props := &content.Schema.Properties + seenNames := map[string]struct{}{} // set equivalent in Go, yikes + for _, pwb := range parts { + name := pwb.part.FormName() + seenNames[name] = struct{}{} + existing, found := (*props)[name] + if !found { + existing = new(openapi.SchemaObj) + existing.Type = openapi.Types{openapi.TypeString} + (*props)[name] = existing + + ctype := pwb.part.Header.Get("content-type") + if ctype != "" { + if existing.Keywords == nil { + existing.Keywords = map[string]json.RawMessage{} + } + existing.Keywords["contentMediaType"], _ = json.Marshal(ctype) + } + } + + addSchemaExample(existing, string(pwb.body)) + } + + // handle required flag + if content.Schema.Required == nil { + if !hadSchema { + content.Schema.Required = make([]string, 0) + for name := range seenNames { + content.Schema.Required = append(content.Schema.Required, name) + } + } // else it's a known schema with no required fields + } else { + content.Schema.Required = intersectSliceWithMap(content.Schema.Required, seenNames) + } +} + +type PartWithBody struct { + part *multipart.Part + body []byte +} + +func handleFormDataMultipart(text string, content *openapi.MediaType, ctypeParams map[string]string) { + boundary, ok := ctypeParams["boundary"] + if !ok { + logger.Log.Errorf("Multipart header has no boundary") + return + } + mpr := multipart.NewReader(strings.NewReader(text), boundary) + + parts := make([]PartWithBody, 0) + for { + part, err := mpr.NextPart() + if err == io.EOF { + break + } + if err != nil { + logger.Log.Errorf("Cannot parse multipart body: %v", err) + break + } + defer part.Close() + + body, err := ioutil.ReadAll(part) + if err != nil { + logger.Log.Errorf("Error reading multipart Part %s: %v", part.Header, err) + } + + parts = append(parts, PartWithBody{part: part, body: body}) + } + + handleFormData(content, parts) +} + func getRespCtype(resp *har.Response) string { var ctype string ctype = resp.Content.MimeType @@ -493,8 +595,7 @@ func getRespCtype(resp *har.Response) string { return mediaType } -func getReqCtype(req *har.Request) string { - var ctype string +func getReqCtype(req *har.Request) (ctype string, params map[string]string) { ctype = req.PostData.MimeType for _, hdr := range req.Headers { if strings.ToLower(hdr.Name) == "content-type" { @@ -502,11 +603,12 @@ func getReqCtype(req *har.Request) string { } } - mediaType, _, err := mime.ParseMediaType(ctype) + mediaType, params, err := mime.ParseMediaType(ctype) if err != nil { - return "" + logger.Log.Errorf("Cannot parse Content-Type header %q: %v", ctype, err) + return "", map[string]string{} } - return mediaType + return mediaType, params } func getResponseObj(resp *har.Response, opObj *openapi.Operation, isSuccess bool) (*openapi.ResponseObj, error) { @@ -591,56 +693,3 @@ func getOpObj(pathObj *openapi.PathObj, method string, createIfNone bool) (*open return *op, isMissing, nil } - -type CounterMaps struct { - counterTotal Counter - counterMapTotal CounterMap -} - -func (m *CounterMaps) processOp(opObj *openapi.Operation) error { - if _, ok := opObj.Extensions.Extension(CountersTotal); ok { - counter := new(Counter) - err := opObj.Extensions.DecodeExtension(CountersTotal, counter) - if err != nil { - return err - } - m.counterTotal.addOther(counter) - - opObj.Description = setCounterMsgIfOk(opObj.Description, counter) - } - - if _, ok := opObj.Extensions.Extension(CountersPerSource); ok { - counterMap := new(CounterMap) - err := opObj.Extensions.DecodeExtension(CountersPerSource, counterMap) - if err != nil { - return err - } - m.counterMapTotal.addOther(counterMap) - } - return nil -} - -func (m *CounterMaps) processOas(oas *openapi.OpenAPI) error { - if oas.Extensions == nil { - oas.Extensions = openapi.Extensions{} - } - - err := oas.Extensions.SetExtension(CountersTotal, m.counterTotal) - if err != nil { - return err - } - - err = oas.Extensions.SetExtension(CountersPerSource, m.counterMapTotal) - if err != nil { - return nil - } - return nil -} - -func setCounterMsgIfOk(oldStr string, cnt *Counter) string { - tpl := "Mizu observed %d entries (%d failed), at %.3f hits/s, average response time is %.3f seconds" - if oldStr == "" || (strings.HasPrefix(oldStr, "Mizu ") && strings.HasSuffix(oldStr, " seconds")) { - return fmt.Sprintf(tpl, cnt.Entries, cnt.Failures, cnt.SumDuration/float64(cnt.Entries), cnt.SumRT/float64(cnt.Entries)) - } - return oldStr -} diff --git a/agent/pkg/oas/specgen_test.go b/agent/pkg/oas/specgen_test.go index f60348e9d..fc59aee88 100644 --- a/agent/pkg/oas/specgen_test.go +++ b/agent/pkg/oas/specgen_test.go @@ -2,20 +2,22 @@ package oas import ( "encoding/json" + "github.com/chanced/openapi" + "github.com/op/go-logging" + "github.com/up9inc/mizu/shared/logger" + "github.com/wI2L/jsondiff" "io/ioutil" "os" + "regexp" "strings" "testing" "time" - "github.com/chanced/openapi" - "github.com/op/go-logging" "github.com/up9inc/mizu/agent/pkg/har" - "github.com/up9inc/mizu/shared/logger" ) // if started via env, write file into subdir -func outputSpec(label string, spec *openapi.OpenAPI, t *testing.T) { +func outputSpec(label string, spec *openapi.OpenAPI, t *testing.T) string { content, err := json.MarshalIndent(spec, "", "\t") if err != nil { panic(err) @@ -35,6 +37,7 @@ func outputSpec(label string, spec *openapi.OpenAPI, t *testing.T) { } else { t.Logf("%s", string(content)) } + return string(content) } func TestEntries(t *testing.T) { @@ -103,6 +106,7 @@ func TestEntries(t *testing.T) { func TestFileSingle(t *testing.T) { GetOasGeneratorInstance().Start() + GetOasGeneratorInstance().Reset() // loadStartingOAS() file := "test_artifacts/params.har" files := []string{file} @@ -123,7 +127,7 @@ func TestFileSingle(t *testing.T) { t.FailNow() } - outputSpec(svc, spec, t) + specText := outputSpec(svc, spec, t) err = spec.Validate() if err != nil { @@ -131,6 +135,29 @@ func TestFileSingle(t *testing.T) { t.FailNow() } + expected, err := ioutil.ReadFile(file + ".spec.json") + if err != nil { + t.Errorf(err.Error()) + t.FailNow() + } + + patFloatPrecision := regexp.MustCompile(`(\d+\.\d{1,2})(\d*)`) + + expected = []byte(patUuid4.ReplaceAllString(string(expected), "")) + specText = patUuid4.ReplaceAllString(specText, "") + expected = []byte(patFloatPrecision.ReplaceAllString(string(expected), "$1")) + specText = patFloatPrecision.ReplaceAllString(specText, "$1") + + diff, err := jsondiff.CompareJSON(expected, []byte(specText)) + if err != nil { + t.Errorf(err.Error()) + t.FailNow() + } + + if len(diff) > 0 { + t.Errorf("Generated spec does not match expected:\n%s", diff.String()) + } + return true }) diff --git a/agent/pkg/oas/test_artifacts/params.har b/agent/pkg/oas/test_artifacts/params.har index f387f3b80..551f4db73 100644 --- a/agent/pkg/oas/test_artifacts/params.har +++ b/agent/pkg/oas/test_artifacts/params.har @@ -235,6 +235,141 @@ "connect": 262, "ssl": -1 } + }, + { + "startedDateTime": "2019-09-06T06:16:20.747122+00:00", + "time": 1, + "request": { + "method": "POST", + "url": "https://httpbin.org/form-urlencoded", + "httpVersion": "", + "cookies": [], + "headers": [ + { + "name": "Content-Type", + "value": "application/x-www-form-urlencoded" + } + ], + "queryString": [], + "headersSize": -1, + "bodySize": -1, + "postData": { + "mimeType": "", + "text": "agent-id=ade&callback-url=&token=sometoken" + } + }, + "response": { + "status": 200, + "statusText": "OK", + "httpVersion": "", + "cookies": [], + "headers": [ + ], + "content": { + "size": 0, + "mimeType": "", + "text": "" + }, + "redirectURL": "", + "headersSize": -1, + "bodySize": 0 + }, + "cache": {}, + "timings": { + "send": -1, + "wait": -1, + "receive": 1 + } + }, + { + "startedDateTime": "2019-09-06T06:16:21.747122+00:00", + "time": 1, + "request": { + "method": "POST", + "url": "https://httpbin.org/form-urlencoded", + "httpVersion": "", + "cookies": [], + "headers": [ + { + "name": "Content-Type", + "value": "application/x-www-form-urlencoded" + } + ], + "queryString": [], + "headersSize": -1, + "bodySize": -1, + "postData": { + "mimeType": "", + "text": "agent-id=ade&callback-url=&token=sometoken-second-val&optional=another" + } + }, + "response": { + "status": 200, + "statusText": "OK", + "httpVersion": "", + "cookies": [], + "headers": [ + ], + "content": { + "size": 0, + "mimeType": "", + "text": "" + }, + "redirectURL": "", + "headersSize": -1, + "bodySize": 0 + }, + "cache": {}, + "timings": { + "send": -1, + "wait": -1, + "receive": 1 + } + }, + { + "startedDateTime": "2019-09-06T06:16:22.747122+00:00", + "time": 1, + "request": { + "method": "POST", + "url": "https://httpbin.org/form-multipart", + "httpVersion": "", + "cookies": [], + "headers": [ + { + "name": "Content-Type", + "value": "multipart/form-data; boundary=BOUNDARY" + } + ], + "queryString": [], + "headersSize": -1, + "bodySize": -1, + "postData": { + "mimeType": "", + "text": "--BOUNDARY\r\nContent-Disposition: form-data; name=\"file\"; filename=\"metadata.json\"\r\nContent-Type: application/json\r\n\r\n{\"functions\": 123}\r\n--BOUNDARY\r\nContent-Disposition: form-data; name=\"path\"\r\n\r\n/content/components\r\n--BOUNDARY--\r\n" + } + }, + "response": { + "status": 200, + "statusText": "OK", + "httpVersion": "", + "cookies": [], + "headers": [ + ], + "content": { + "size": 62, + "mimeType": "", + "text": "{}" + }, + "redirectURL": "", + "headersSize": -1, + "bodySize": 62 + }, + "cache": {}, + "timings": { + "send": -1, + "wait": -1, + "receive": 1 + } } ] } diff --git a/agent/pkg/oas/test_artifacts/params.har.spec.json b/agent/pkg/oas/test_artifacts/params.har.spec.json new file mode 100644 index 000000000..28cb200ba --- /dev/null +++ b/agent/pkg/oas/test_artifacts/params.har.spec.json @@ -0,0 +1,413 @@ +{ + "openapi": "3.1.0", + "info": { + "title": "https://httpbin.org", + "description": "Mizu observed 9 entries (0 failed), at 0.222 hits/s, average response time is 0.363 seconds", + "version": "1.0" + }, + "servers": [ + { + "url": "https://httpbin.org" + } + ], + "paths": { + "/appears-once": { + "get": { + "summary": "/appears-once", + "description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.630 seconds", + "operationId": "ebf78fe8-6ebe-40bb-ad88-eab0dce05e91", + "responses": { + "200": { + "description": "Successful call with status 200", + "content": { + "application/json": { + "example": null + } + } + } + }, + "x-last-seen-ts": 1567750580.0471218, + "x-counters-total": { + "entries": 1, + "failures": 0, + "firstSeen": 1567750580.0471218, + "lastSeen": 1567750580.0471218, + "sumRT": 0.63, + "sumDuration": 0 + }, + "x-counters-per-source": { + "": { + "entries": 1, + "failures": 0, + "firstSeen": 1567750580.0471218, + "lastSeen": 1567750580.0471218, + "sumRT": 0.63, + "sumDuration": 0 + } + } + } + }, + "/appears-twice": { + "get": { + "summary": "/appears-twice", + "description": "Mizu observed 2 entries (0 failed), at 0.500 hits/s, average response time is 0.630 seconds", + "operationId": "67e6640a-8cb2-4e31-ae4d-b066b397ee93", + "responses": { + "200": { + "description": "Successful call with status 200", + "content": { + "application/json": { + "example": null + } + } + } + }, + "x-last-seen-ts": 1567750581.7471218, + "x-counters-total": { + "entries": 2, + "failures": 0, + "firstSeen": 1567750580.7471218, + "lastSeen": 1567750581.7471218, + "sumRT": 1.26, + "sumDuration": 1 + }, + "x-counters-per-source": { + "": { + "entries": 2, + "failures": 0, + "firstSeen": 1567750580.7471218, + "lastSeen": 1567750581.7471218, + "sumRT": 1.26, + "sumDuration": 1 + } + } + } + }, + "/form-multipart": { + "post": { + "summary": "/form-multipart", + "description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.001 seconds", + "operationId": "a8e0380e-3ade-4150-af6b-9edb7f24d9dc", + "responses": { + "200": { + "description": "Successful call with status 200", + "content": { + "": { + "example": {} + } + } + } + }, + "x-counters-total": { + "entries": 1, + "failures": 0, + "firstSeen": 1567750582.7471218, + "lastSeen": 1567750582.7471218, + "sumRT": 0.001, + "sumDuration": 0 + }, + "x-counters-per-source": { + "": { + "entries": 1, + "failures": 0, + "firstSeen": 1567750582.7471218, + "lastSeen": 1567750582.7471218, + "sumRT": 0.001, + "sumDuration": 0 + } + }, + "x-last-seen-ts": 1567750582.7471218, + "requestBody": { + "description": "Generic request body", + "content": { + "multipart/form-data": { + "schema": { + "type": "object", + "required": [ + "file", + "path" + ], + "properties": { + "file": { + "type": "string", + "contentMediaType": "application/json", + "examples": [ + "{\"functions\": 123}" + ] + }, + "path": { + "type": "string", + "examples": [ + "/content/components" + ] + } + } + }, + "example": "--BOUNDARY\r\nContent-Disposition: form-data; name=\"file\"; filename=\"metadata.json\"\r\nContent-Type: application/json\r\n\r\n{\"functions\": 123}\r\n--BOUNDARY\r\nContent-Disposition: form-data; name=\"path\"\r\n\r\n/content/components\r\n--BOUNDARY--\r\n" + } + }, + "required": true + } + } + }, + "/form-urlencoded": { + "post": { + "summary": "/form-urlencoded", + "description": "Mizu observed 2 entries (0 failed), at 0.500 hits/s, average response time is 0.001 seconds", + "operationId": "dd251f61-2636-4c38-ad9d-d4b654464eba", + "responses": { + "200": { + "description": "Successful call with status 200", + "content": { + "": {} + } + } + }, + "x-last-seen-ts": 1567750581.7471218, + "x-counters-total": { + "entries": 2, + "failures": 0, + "firstSeen": 1567750580.7471218, + "lastSeen": 1567750581.7471218, + "sumRT": 0.002, + "sumDuration": 1 + }, + "x-counters-per-source": { + "": { + "entries": 2, + "failures": 0, + "firstSeen": 1567750580.7471218, + "lastSeen": 1567750581.7471218, + "sumRT": 0.002, + "sumDuration": 1 + } + }, + "requestBody": { + "description": "Generic request body", + "content": { + "application/x-www-form-urlencoded": { + "schema": { + "type": "object", + "required": [ + "agent-id", + "callback-url", + "token" + ], + "properties": { + "agent-id": { + "type": "string", + "examples": [ + "ade" + ] + }, + "callback-url": { + "type": "string", + "examples": [ + "" + ] + }, + "optional": { + "type": "string", + "examples": [ + "another" + ] + }, + "token": { + "type": "string", + "examples": [ + "sometoken", + "sometoken-second-val" + ] + } + } + }, + "example": "agent-id=ade\u0026callback-url=\u0026token=sometoken" + } + }, + "required": true + } + } + }, + "/{Id}": { + "get": { + "summary": "/{Id}", + "description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.630 seconds", + "operationId": "095b3932-c824-4054-a6da-bc872f279743", + "responses": { + "200": { + "description": "Successful call with status 200", + "content": { + "application/json": { + "example": null + } + } + } + }, + "x-last-seen-ts": 1567750579.7471218, + "x-counters-total": { + "entries": 1, + "failures": 0, + "firstSeen": 1567750579.7471218, + "lastSeen": 1567750579.7471218, + "sumRT": 0.63, + "sumDuration": 0 + }, + "x-counters-per-source": { + "": { + "entries": 1, + "failures": 0, + "firstSeen": 1567750579.7471218, + "lastSeen": 1567750579.7471218, + "sumRT": 0.63, + "sumDuration": 0 + } + } + }, + "parameters": [ + { + "name": "Id", + "in": "path", + "required": true, + "style": "simple", + "schema": { + "type": "string" + }, + "examples": { + "example #0": { + "value": "e21f7112-3d3b-4632-9da3-a4af2e0e9166" + }, + "example #1": { + "value": "952bea17-3776-11ea-9341-42010a84012a" + } + } + } + ] + }, + "/{Id}/sub1": { + "get": { + "summary": "/{Id}/sub1", + "description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.111 seconds", + "operationId": "6f168b6d-9b67-490e-9895-903b078de4c5", + "responses": { + "200": { + "description": "Successful call with status 200", + "content": { + "text/html": {} + } + } + }, + "x-last-seen-ts": 1567750483.864529, + "x-counters-total": { + "entries": 1, + "failures": 0, + "firstSeen": 1567750483.864529, + "lastSeen": 1567750483.864529, + "sumRT": 0.111, + "sumDuration": 0 + }, + "x-counters-per-source": { + "": { + "entries": 1, + "failures": 0, + "firstSeen": 1567750483.864529, + "lastSeen": 1567750483.864529, + "sumRT": 0.111, + "sumDuration": 0 + } + } + }, + "parameters": [ + { + "name": "Id", + "in": "path", + "required": true, + "style": "simple", + "schema": { + "type": "string" + }, + "examples": { + "example #0": { + "value": "e21f7112-3d3b-4632-9da3-a4af2e0e9166" + }, + "example #1": { + "value": "952bea17-3776-11ea-9341-42010a84012a" + } + } + } + ] + }, + "/{Id}/sub2": { + "get": { + "summary": "/{Id}/sub2", + "description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.630 seconds", + "operationId": "d1e7900f-01dc-4d79-912e-5ca79ecd73e3", + "responses": { + "200": { + "description": "Successful call with status 200", + "content": { + "application/json": { + "example": null + } + } + } + }, + "x-counters-per-source": { + "": { + "entries": 1, + "failures": 0, + "firstSeen": 1567750578.7471218, + "lastSeen": 1567750578.7471218, + "sumRT": 0.63, + "sumDuration": 0 + } + }, + "x-last-seen-ts": 1567750578.7471218, + "x-counters-total": { + "entries": 1, + "failures": 0, + "firstSeen": 1567750578.7471218, + "lastSeen": 1567750578.7471218, + "sumRT": 0.63, + "sumDuration": 0 + } + }, + "parameters": [ + { + "name": "Id", + "in": "path", + "required": true, + "style": "simple", + "schema": { + "type": "string" + }, + "examples": { + "example #0": { + "value": "e21f7112-3d3b-4632-9da3-a4af2e0e9166" + }, + "example #1": { + "value": "952bea17-3776-11ea-9341-42010a84012a" + } + } + } + ] + } + }, + "x-counters-total": { + "entries": 9, + "failures": 0, + "firstSeen": 1567750483.864529, + "lastSeen": 1567750582.7471218, + "sumRT": 3.2639999999999993, + "sumDuration": 2 + }, + "x-counters-per-source": { + "": { + "entries": 9, + "failures": 0, + "firstSeen": 1567750483.864529, + "lastSeen": 1567750582.7471218, + "sumRT": 3.2639999999999993, + "sumDuration": 2 + } + } +} \ No newline at end of file diff --git a/agent/pkg/oas/utils.go b/agent/pkg/oas/utils.go index 0c7b375f2..7d95f3b14 100644 --- a/agent/pkg/oas/utils.go +++ b/agent/pkg/oas/utils.go @@ -52,8 +52,7 @@ func createSimpleParam(name string, in openapi.In, ptype openapi.SchemaType) *op } required := true // FFS! https://stackoverflow.com/questions/32364027/reference-a-boolean-for-assignment-in-a-struct/32364093 schema := new(openapi.SchemaObj) - schema.Type = make(openapi.Types, 0) - schema.Type = append(schema.Type, ptype) + schema.Type = openapi.Types{ptype} style := openapi.StyleSimple if in == openapi.InQuery { @@ -197,7 +196,7 @@ func fillParamExample(param **openapi.Examples, exampleValue string) error { continue } - if value == exampleValue || cnt > 5 { // 5 examples is enough + if value == exampleValue || cnt >= 5 { // 5 examples is enough return nil } } @@ -213,6 +212,36 @@ func fillParamExample(param **openapi.Examples, exampleValue string) error { return nil } +// TODO: somehow generalize the two example setting functions, plus add body example handling + +func addSchemaExample(existing *openapi.SchemaObj, bodyStr string) { + if len(existing.Examples) < 5 { + found := false + for _, eVal := range existing.Examples { + existingExample := "" + err := json.Unmarshal(eVal, &existingExample) + if err != nil { + logger.Log.Debugf("Failed to unmarshal example: %v", eVal) + continue + } + + if existingExample == bodyStr { + found = true + break + } + } + + if !found { + example, err := json.Marshal(bodyStr) + if err != nil { + logger.Log.Debugf("Failed to marshal example: %v", bodyStr) + return + } + existing.Examples = append(existing.Examples, example) + } + } +} + func longestCommonXfix(strs [][]string, pre bool) []string { // https://github.com/jpillora/longestcommon empty := make([]string, 0) //short-circuit empty list @@ -370,3 +399,31 @@ func isAlphaRune(r rune) bool { func isAlNumRune(b rune) bool { return isAlphaRune(b) || ('0' <= b && b <= '9') } + +func deleteFromSlice(s []string, val string) []string { + temp := s[:0] + for _, x := range s { + if x != val { + temp = append(temp, x) + } + } + return temp +} + +func sliceContains(s []string, e string) bool { + for _, a := range s { + if a == e { + return true + } + } + return false +} + +func intersectSliceWithMap(required []string, names map[string]struct{}) []string { + for name := range names { + if !sliceContains(required, name) { + required = deleteFromSlice(required, name) + } + } + return required +}