github/kubeshark
1
0
Fork 0
mirror of https://github.com/kubeshark/kubeshark.git synced 2025-06-21 22:08:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

OAS: handle urlencoded and multipart form data correctly (#730)

Browse Source 
* Start with tests

* Expected file

* Almost works

* Make test stab;e

* 5 examples, not 6

* Add param type

* multipart example

* parsing multipart

* Commit

* Commit

* multipart

* Write encoding into schema

* Stable test

* Add reset

* Refactoring

* Refactoring

* Maintain the required flag

* lint
This commit is contained in:
Andrey Pokhilko 2022-02-03 11:46:31 +03:00 committed by GitHub
parent 78be20fe4d
commit 5934be4da6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 833 additions and 88 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
agent/go.mod
View File

@ -4,7 +4,7 @@ go 1.16
require (
github.com/antelman107/net-wait-go v0.0.0-20210623112055-cf684aebda7b
github.com/chanced/openapi v0.0.6
github.com/chanced/openapi v0.0.7
github.com/djherbis/atime v1.0.0
github.com/elastic/go-elasticsearch/v7 v7.16.0
github.com/getkin/kin-openapi v0.76.0
@ -30,6 +30,7 @@ require (
github.com/up9inc/mizu/tap/extensions/http v0.0.0
github.com/up9inc/mizu/tap/extensions/kafka v0.0.0
github.com/up9inc/mizu/tap/extensions/redis v0.0.0
github.com/wI2L/jsondiff v0.1.1
github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0
k8s.io/api v0.21.2
k8s.io/apimachinery v0.21.2

4
agent/go.sum
View File

@ -95,8 +95,8 @@ github.com/chanced/cmpjson v0.0.0-20210415035445-da9262c1f20a h1:zG6t+4krPXcCKtL
github.com/chanced/cmpjson v0.0.0-20210415035445-da9262c1f20a/go.mod h1:yhcmlFk1hxuZ+5XZbupzT/cEm/eE4ZvWbmsW1+Q/aZE=
github.com/chanced/dynamic v0.0.0-20210502140838-c010b5fc3e44 h1:4NOJMtvZaOA6cI2gkIuXk/2b5KTOvm/R4zyPy/yLCM4=
github.com/chanced/dynamic v0.0.0-20210502140838-c010b5fc3e44/go.mod h1:XVNfXN5kgZST4PQ0W/oBAHJku2OteCeHxjAbvfd0ARM=
github.com/chanced/openapi v0.0.6 h1:2giGS47+T8/7MN2hfRHSIUPx86Qksk+J/ciIWcAM7hY=
github.com/chanced/openapi v0.0.6/go.mod h1:SxE2VMLPw+T7Vq8nwbVVhDF2PigvRF4n5XyqsVpRJGU=
github.com/chanced/openapi v0.0.7 h1:OmOBHCg/5ViUg0gaGxXBeEFoVBE8C2pHK4BO/AiD6k8=
github.com/chanced/openapi v0.0.7/go.mod h1:SxE2VMLPw+T7Vq8nwbVVhDF2PigvRF4n5XyqsVpRJGU=
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=

60
agent/pkg/oas/counters.go
View File

@ -1,6 +1,11 @@
package oas
import "math"
import (
"fmt"
"github.com/chanced/openapi"
"math"
"strings"
)
type Counter struct {
Entries int `json:"entries"`
@ -55,3 +60,56 @@ func (m *CounterMap) addOther(other *CounterMap) {
}
}
}
func setCounterMsgIfOk(oldStr string, cnt *Counter) string {
tpl := "Mizu observed %d entries (%d failed), at %.3f hits/s, average response time is %.3f seconds"
if oldStr == "" || (strings.HasPrefix(oldStr, "Mizu ") && strings.HasSuffix(oldStr, " seconds")) {
return fmt.Sprintf(tpl, cnt.Entries, cnt.Failures, cnt.SumDuration/float64(cnt.Entries), cnt.SumRT/float64(cnt.Entries))
}
return oldStr
}
type CounterMaps struct {
counterTotal Counter
counterMapTotal CounterMap
}
func (m *CounterMaps) processOp(opObj *openapi.Operation) error {
if _, ok := opObj.Extensions.Extension(CountersTotal); ok {
counter := new(Counter)
err := opObj.Extensions.DecodeExtension(CountersTotal, counter)
if err != nil {
return err
}
m.counterTotal.addOther(counter)
opObj.Description = setCounterMsgIfOk(opObj.Description, counter)
}
if _, ok := opObj.Extensions.Extension(CountersPerSource); ok {
counterMap := new(CounterMap)
err := opObj.Extensions.DecodeExtension(CountersPerSource, counterMap)
if err != nil {
return err
}
m.counterMapTotal.addOther(counterMap)
}
return nil
}
func (m *CounterMaps) processOas(oas *openapi.OpenAPI) error {
if oas.Extensions == nil {
oas.Extensions = openapi.Extensions{}
}
err := oas.Extensions.SetExtension(CountersTotal, m.counterTotal)
if err != nil {
return err
}
err = oas.Extensions.SetExtension(CountersPerSource, m.counterMapTotal)
if err != nil {
return nil
}
return nil
}

7
agent/pkg/oas/feeder_test.go
View File

@ -110,13 +110,14 @@ func feedFromHAR(file string, isSync bool) (int, error) {
cnt := 0
for _, entry := range harDoc.Log.Entries {
cnt += 1
feedEntry(&entry, "", isSync)
feedEntry(&entry, "", isSync, file)
}
return cnt, nil
}
func feedEntry(entry *har.Entry, source string, isSync bool) {
func feedEntry(entry *har.Entry, source string, isSync bool, file string) {
entry.Comment = file
if entry.Response.Status == 302 {
logger.Log.Debugf("Dropped traffic entry due to permanent redirect status: %s", entry.StartedDateTime)
}
@ -176,7 +177,7 @@ func feedFromLDJSON(file string, isSync bool) (int, error) {
logger.Log.Warningf("Failed decoding entry: %s", line)
} else {
cnt += 1
feedEntry(&entry, source, isSync)
feedEntry(&entry, source, isSync, file)
}
}
}

4
agent/pkg/oas/oas_generator.go
View File

@ -79,6 +79,10 @@ func (g *oasGenerator) runGeneretor() {
}
}
func (g *oasGenerator) Reset() {
g.ServiceSpecs = &sync.Map{}
}
func (g *oasGenerator) PushEntry(entryWithSource *EntryWithSource) {
if !g.started {
return

195
agent/pkg/oas/specgen.go
View File

@ -3,13 +3,17 @@ package oas
import (
"encoding/json"
"errors"
"fmt"
"github.com/chanced/openapi"
"github.com/google/uuid"
"github.com/nav-inc/datetime"
"github.com/up9inc/mizu/shared/logger"
"io"
"io/ioutil"
"mime"
"mime/multipart"
"net/textproto"
"net/url"
"sort"
"strconv"
"strings"
"sync"
@ -126,6 +130,7 @@ func (g *SpecGen) GetSpec() (*openapi.OpenAPI, error) {
func suggestTags(oas *openapi.OpenAPI) {
paths := getPathsKeys(oas.Paths.Items)
sort.Strings(paths) // make it stable in case of multiple candidates
for len(paths) > 0 {
group := make([]string, 0)
group = append(group, paths[0])
@ -155,21 +160,9 @@ func suggestTags(oas *openapi.OpenAPI) {
}
}
}
//groups[common] = group
}
}
func deleteFromSlice(s []string, val string) []string {
temp := s[:0]
for _, x := range s {
if x != val {
temp = append(temp, x)
}
}
return temp
}
func getPathsKeys(mymap map[openapi.PathValue]*openapi.PathObj) []string {
keys := make([]string, len(mymap))
@ -364,7 +357,7 @@ func handleRequest(req *har.Request, opObj *openapi.Operation, isSuccess bool) e
}
if reqBody != nil {
reqCtype := getReqCtype(req)
reqCtype, _ := getReqCtype(req)
reqMedia, err := fillContent(reqResp{Req: req}, reqBody.Content, reqCtype)
if err != nil {
return err
@ -471,12 +464,121 @@ func fillContent(reqResp reqResp, respContent openapi.Content, ctype string) (*o
}
}
content.Example = exampleMsg
if ctype == "application/x-www-form-urlencoded" && reqResp.Req != nil {
handleFormDataUrlencoded(text, content)
} else if strings.HasPrefix(ctype, "multipart/form-data") && reqResp.Req != nil {
_, params := getReqCtype(reqResp.Req)
handleFormDataMultipart(text, content, params)
}
if content.Example == nil && len(exampleMsg) > len(content.Example) {
content.Example = exampleMsg
}
}
return respContent[ctype], nil
}
func handleFormDataUrlencoded(text string, content *openapi.MediaType) {
formData, err := url.ParseQuery(text)
if err != nil {
logger.Log.Warningf("Could not decode urlencoded: %s", err)
return
}
parts := make([]PartWithBody, 0)
for name, vals := range formData {
for _, val := range vals {
part := new(multipart.Part)
part.Header = textproto.MIMEHeader{}
part.Header.Add("Content-Disposition", "form-data; name=\""+name+"\";")
parts = append(parts, PartWithBody{part: part, body: []byte(val)})
}
}
handleFormData(content, parts)
}
func handleFormData(content *openapi.MediaType, parts []PartWithBody) {
hadSchema := true
if content.Schema == nil {
hadSchema = false // will use it for required flags
content.Schema = new(openapi.SchemaObj)
content.Schema.Type = openapi.Types{openapi.TypeObject}
content.Schema.Properties = openapi.Schemas{}
}
props := &content.Schema.Properties
seenNames := map[string]struct{}{} // set equivalent in Go, yikes
for _, pwb := range parts {
name := pwb.part.FormName()
seenNames[name] = struct{}{}
existing, found := (*props)[name]
if !found {
existing = new(openapi.SchemaObj)
existing.Type = openapi.Types{openapi.TypeString}
(*props)[name] = existing
ctype := pwb.part.Header.Get("content-type")
if ctype != "" {
if existing.Keywords == nil {
existing.Keywords = map[string]json.RawMessage{}
}
existing.Keywords["contentMediaType"], _ = json.Marshal(ctype)
}
}
addSchemaExample(existing, string(pwb.body))
}
// handle required flag
if content.Schema.Required == nil {
if !hadSchema {
content.Schema.Required = make([]string, 0)
for name := range seenNames {
content.Schema.Required = append(content.Schema.Required, name)
}
} // else it's a known schema with no required fields
} else {
content.Schema.Required = intersectSliceWithMap(content.Schema.Required, seenNames)
}
}
type PartWithBody struct {
part *multipart.Part
body []byte
}
func handleFormDataMultipart(text string, content *openapi.MediaType, ctypeParams map[string]string) {
boundary, ok := ctypeParams["boundary"]
if !ok {
logger.Log.Errorf("Multipart header has no boundary")
return
}
mpr := multipart.NewReader(strings.NewReader(text), boundary)
parts := make([]PartWithBody, 0)
for {
part, err := mpr.NextPart()
if err == io.EOF {
break
}
if err != nil {
logger.Log.Errorf("Cannot parse multipart body: %v", err)
break
}
defer part.Close()
body, err := ioutil.ReadAll(part)
if err != nil {
logger.Log.Errorf("Error reading multipart Part %s: %v", part.Header, err)
}
parts = append(parts, PartWithBody{part: part, body: body})
}
handleFormData(content, parts)
}
func getRespCtype(resp *har.Response) string {
var ctype string
ctype = resp.Content.MimeType
@ -493,8 +595,7 @@ func getRespCtype(resp *har.Response) string {
return mediaType
}
func getReqCtype(req *har.Request) string {
var ctype string
func getReqCtype(req *har.Request) (ctype string, params map[string]string) {
ctype = req.PostData.MimeType
for _, hdr := range req.Headers {
if strings.ToLower(hdr.Name) == "content-type" {
@ -502,11 +603,12 @@ func getReqCtype(req *har.Request) string {
}
}
mediaType, _, err := mime.ParseMediaType(ctype)
mediaType, params, err := mime.ParseMediaType(ctype)
if err != nil {
return ""
logger.Log.Errorf("Cannot parse Content-Type header %q: %v", ctype, err)
return "", map[string]string{}
}
return mediaType
return mediaType, params
}
func getResponseObj(resp *har.Response, opObj *openapi.Operation, isSuccess bool) (*openapi.ResponseObj, error) {
@ -591,56 +693,3 @@ func getOpObj(pathObj *openapi.PathObj, method string, createIfNone bool) (*open
return *op, isMissing, nil
}
type CounterMaps struct {
counterTotal Counter
counterMapTotal CounterMap
}
func (m *CounterMaps) processOp(opObj *openapi.Operation) error {
if _, ok := opObj.Extensions.Extension(CountersTotal); ok {
counter := new(Counter)
err := opObj.Extensions.DecodeExtension(CountersTotal, counter)
if err != nil {
return err
}
m.counterTotal.addOther(counter)
opObj.Description = setCounterMsgIfOk(opObj.Description, counter)
}
if _, ok := opObj.Extensions.Extension(CountersPerSource); ok {
counterMap := new(CounterMap)
err := opObj.Extensions.DecodeExtension(CountersPerSource, counterMap)
if err != nil {
return err
}
m.counterMapTotal.addOther(counterMap)
}
return nil
}
func (m *CounterMaps) processOas(oas *openapi.OpenAPI) error {
if oas.Extensions == nil {
oas.Extensions = openapi.Extensions{}
}
err := oas.Extensions.SetExtension(CountersTotal, m.counterTotal)
if err != nil {
return err
}
err = oas.Extensions.SetExtension(CountersPerSource, m.counterMapTotal)
if err != nil {
return nil
}
return nil
}
func setCounterMsgIfOk(oldStr string, cnt *Counter) string {
tpl := "Mizu observed %d entries (%d failed), at %.3f hits/s, average response time is %.3f seconds"
if oldStr == "" || (strings.HasPrefix(oldStr, "Mizu ") && strings.HasSuffix(oldStr, " seconds")) {
return fmt.Sprintf(tpl, cnt.Entries, cnt.Failures, cnt.SumDuration/float64(cnt.Entries), cnt.SumRT/float64(cnt.Entries))
}
return oldStr
}

37
agent/pkg/oas/specgen_test.go
View File

@ -2,20 +2,22 @@ package oas
import (
"encoding/json"
"github.com/chanced/openapi"
"github.com/op/go-logging"
"github.com/up9inc/mizu/shared/logger"
"github.com/wI2L/jsondiff"
"io/ioutil"
"os"
"regexp"
"strings"
"testing"
"time"
"github.com/chanced/openapi"
"github.com/op/go-logging"
"github.com/up9inc/mizu/agent/pkg/har"
"github.com/up9inc/mizu/shared/logger"
)
// if started via env, write file into subdir
func outputSpec(label string, spec *openapi.OpenAPI, t *testing.T) {
func outputSpec(label string, spec *openapi.OpenAPI, t *testing.T) string {
content, err := json.MarshalIndent(spec, "", "\t")
if err != nil {
panic(err)
@ -35,6 +37,7 @@ func outputSpec(label string, spec *openapi.OpenAPI, t *testing.T) {
} else {
t.Logf("%s", string(content))
}
return string(content)
}
func TestEntries(t *testing.T) {
@ -103,6 +106,7 @@ func TestEntries(t *testing.T) {
func TestFileSingle(t *testing.T) {
GetOasGeneratorInstance().Start()
GetOasGeneratorInstance().Reset()
// loadStartingOAS()
file := "test_artifacts/params.har"
files := []string{file}
@ -123,7 +127,7 @@ func TestFileSingle(t *testing.T) {
t.FailNow()
}
outputSpec(svc, spec, t)
specText := outputSpec(svc, spec, t)
err = spec.Validate()
if err != nil {
@ -131,6 +135,29 @@ func TestFileSingle(t *testing.T) {
t.FailNow()
}
expected, err := ioutil.ReadFile(file + ".spec.json")
if err != nil {
t.Errorf(err.Error())
t.FailNow()
}
patFloatPrecision := regexp.MustCompile(`(\d+\.\d{1,2})(\d*)`)
expected = []byte(patUuid4.ReplaceAllString(string(expected), "<UUID4>"))
specText = patUuid4.ReplaceAllString(specText, "<UUID4>")
expected = []byte(patFloatPrecision.ReplaceAllString(string(expected), "$1"))
specText = patFloatPrecision.ReplaceAllString(specText, "$1")
diff, err := jsondiff.CompareJSON(expected, []byte(specText))
if err != nil {
t.Errorf(err.Error())
t.FailNow()
}
if len(diff) > 0 {
t.Errorf("Generated spec does not match expected:\n%s", diff.String())
}
return true
})

135
agent/pkg/oas/test_artifacts/params.har
View File

@ -235,6 +235,141 @@
"connect": 262,
"ssl": -1
}
},
{
"startedDateTime": "2019-09-06T06:16:20.747122+00:00",
"time": 1,
"request": {
"method": "POST",
"url": "https://httpbin.org/form-urlencoded",
"httpVersion": "",
"cookies": [],
"headers": [
{
"name": "Content-Type",
"value": "application/x-www-form-urlencoded"
}
],
"queryString": [],
"headersSize": -1,
"bodySize": -1,
"postData": {
"mimeType": "",
"text": "agent-id=ade&callback-url=&token=sometoken"
}
},
"response": {
"status": 200,
"statusText": "OK",
"httpVersion": "",
"cookies": [],
"headers": [
],
"content": {
"size": 0,
"mimeType": "",
"text": ""
},
"redirectURL": "",
"headersSize": -1,
"bodySize": 0
},
"cache": {},
"timings": {
"send": -1,
"wait": -1,
"receive": 1
}
},
{
"startedDateTime": "2019-09-06T06:16:21.747122+00:00",
"time": 1,
"request": {
"method": "POST",
"url": "https://httpbin.org/form-urlencoded",
"httpVersion": "",
"cookies": [],
"headers": [
{
"name": "Content-Type",
"value": "application/x-www-form-urlencoded"
}
],
"queryString": [],
"headersSize": -1,
"bodySize": -1,
"postData": {
"mimeType": "",
"text": "agent-id=ade&callback-url=&token=sometoken-second-val&optional=another"
}
},
"response": {
"status": 200,
"statusText": "OK",
"httpVersion": "",
"cookies": [],
"headers": [
],
"content": {
"size": 0,
"mimeType": "",
"text": ""
},
"redirectURL": "",
"headersSize": -1,
"bodySize": 0
},
"cache": {},
"timings": {
"send": -1,
"wait": -1,
"receive": 1
}
},
{
"startedDateTime": "2019-09-06T06:16:22.747122+00:00",
"time": 1,
"request": {
"method": "POST",
"url": "https://httpbin.org/form-multipart",
"httpVersion": "",
"cookies": [],
"headers": [
{
"name": "Content-Type",
"value": "multipart/form-data; boundary=BOUNDARY"
}
],
"queryString": [],
"headersSize": -1,
"bodySize": -1,
"postData": {
"mimeType": "",
"text": "--BOUNDARY\r\nContent-Disposition: form-data; name=\"file\"; filename=\"metadata.json\"\r\nContent-Type: application/json\r\n\r\n{\"functions\": 123}\r\n--BOUNDARY\r\nContent-Disposition: form-data; name=\"path\"\r\n\r\n/content/components\r\n--BOUNDARY--\r\n"
}
},
"response": {
"status": 200,
"statusText": "OK",
"httpVersion": "",
"cookies": [],
"headers": [
],
"content": {
"size": 62,
"mimeType": "",
"text": "{}"
},
"redirectURL": "",
"headersSize": -1,
"bodySize": 62
},
"cache": {},
"timings": {
"send": -1,
"wait": -1,
"receive": 1
}
}
]
}

413
agent/pkg/oas/test_artifacts/params.har.spec.json Normal file
View File

@ -0,0 +1,413 @@
{
"openapi": "3.1.0",
"info": {
"title": "https://httpbin.org",
"description": "Mizu observed 9 entries (0 failed), at 0.222 hits/s, average response time is 0.363 seconds",
"version": "1.0"
},
"servers": [
{
"url": "https://httpbin.org"
}
],
"paths": {
"/appears-once": {
"get": {
"summary": "/appears-once",
"description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.630 seconds",
"operationId": "ebf78fe8-6ebe-40bb-ad88-eab0dce05e91",
"responses": {
"200": {
"description": "Successful call with status 200",
"content": {
"application/json": {
"example": null
}
}
}
},
"x-last-seen-ts": 1567750580.0471218,
"x-counters-total": {
"entries": 1,
"failures": 0,
"firstSeen": 1567750580.0471218,
"lastSeen": 1567750580.0471218,
"sumRT": 0.63,
"sumDuration": 0
},
"x-counters-per-source": {
"": {
"entries": 1,
"failures": 0,
"firstSeen": 1567750580.0471218,
"lastSeen": 1567750580.0471218,
"sumRT": 0.63,
"sumDuration": 0
}
}
}
},
"/appears-twice": {
"get": {
"summary": "/appears-twice",
"description": "Mizu observed 2 entries (0 failed), at 0.500 hits/s, average response time is 0.630 seconds",
"operationId": "67e6640a-8cb2-4e31-ae4d-b066b397ee93",
"responses": {
"200": {
"description": "Successful call with status 200",
"content": {
"application/json": {
"example": null
}
}
}
},
"x-last-seen-ts": 1567750581.7471218,
"x-counters-total": {
"entries": 2,
"failures": 0,
"firstSeen": 1567750580.7471218,
"lastSeen": 1567750581.7471218,
"sumRT": 1.26,
"sumDuration": 1
},
"x-counters-per-source": {
"": {
"entries": 2,
"failures": 0,
"firstSeen": 1567750580.7471218,
"lastSeen": 1567750581.7471218,
"sumRT": 1.26,
"sumDuration": 1
}
}
}
},
"/form-multipart": {
"post": {
"summary": "/form-multipart",
"description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.001 seconds",
"operationId": "a8e0380e-3ade-4150-af6b-9edb7f24d9dc",
"responses": {
"200": {
"description": "Successful call with status 200",
"content": {
"": {
"example": {}
}
}
}
},
"x-counters-total": {
"entries": 1,
"failures": 0,
"firstSeen": 1567750582.7471218,
"lastSeen": 1567750582.7471218,
"sumRT": 0.001,
"sumDuration": 0
},
"x-counters-per-source": {
"": {
"entries": 1,
"failures": 0,
"firstSeen": 1567750582.7471218,
"lastSeen": 1567750582.7471218,
"sumRT": 0.001,
"sumDuration": 0
}
},
"x-last-seen-ts": 1567750582.7471218,
"requestBody": {
"description": "Generic request body",
"content": {
"multipart/form-data": {
"schema": {
"type": "object",
"required": [
"file",
"path"
],
"properties": {
"file": {
"type": "string",
"contentMediaType": "application/json",
"examples": [
"{\"functions\": 123}"
]
},
"path": {
"type": "string",
"examples": [
"/content/components"
]
}
}
},
"example": "--BOUNDARY\r\nContent-Disposition: form-data; name=\"file\"; filename=\"metadata.json\"\r\nContent-Type: application/json\r\n\r\n{\"functions\": 123}\r\n--BOUNDARY\r\nContent-Disposition: form-data; name=\"path\"\r\n\r\n/content/components\r\n--BOUNDARY--\r\n"
}
},
"required": true
}
}
},
"/form-urlencoded": {
"post": {
"summary": "/form-urlencoded",
"description": "Mizu observed 2 entries (0 failed), at 0.500 hits/s, average response time is 0.001 seconds",
"operationId": "dd251f61-2636-4c38-ad9d-d4b654464eba",
"responses": {
"200": {
"description": "Successful call with status 200",
"content": {
"": {}
}
}
},
"x-last-seen-ts": 1567750581.7471218,
"x-counters-total": {
"entries": 2,
"failures": 0,
"firstSeen": 1567750580.7471218,
"lastSeen": 1567750581.7471218,
"sumRT": 0.002,
"sumDuration": 1
},
"x-counters-per-source": {
"": {
"entries": 2,
"failures": 0,
"firstSeen": 1567750580.7471218,
"lastSeen": 1567750581.7471218,
"sumRT": 0.002,
"sumDuration": 1
}
},
"requestBody": {
"description": "Generic request body",
"content": {
"application/x-www-form-urlencoded": {
"schema": {
"type": "object",
"required": [
"agent-id",
"callback-url",
"token"
],
"properties": {
"agent-id": {
"type": "string",
"examples": [
"ade"
]
},
"callback-url": {
"type": "string",
"examples": [
""
]
},
"optional": {
"type": "string",
"examples": [
"another"
]
},
"token": {
"type": "string",
"examples": [
"sometoken",
"sometoken-second-val"
]
}
}
},
"example": "agent-id=ade\u0026callback-url=\u0026token=sometoken"
}
},
"required": true
}
}
},
"/{Id}": {
"get": {
"summary": "/{Id}",
"description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.630 seconds",
"operationId": "095b3932-c824-4054-a6da-bc872f279743",
"responses": {
"200": {
"description": "Successful call with status 200",
"content": {
"application/json": {
"example": null
}
}
}
},
"x-last-seen-ts": 1567750579.7471218,
"x-counters-total": {
"entries": 1,
"failures": 0,
"firstSeen": 1567750579.7471218,
"lastSeen": 1567750579.7471218,
"sumRT": 0.63,
"sumDuration": 0
},
"x-counters-per-source": {
"": {
"entries": 1,
"failures": 0,
"firstSeen": 1567750579.7471218,
"lastSeen": 1567750579.7471218,
"sumRT": 0.63,
"sumDuration": 0
}
}
},
"parameters": [
{
"name": "Id",
"in": "path",
"required": true,
"style": "simple",
"schema": {
"type": "string"
},
"examples": {
"example #0": {
"value": "e21f7112-3d3b-4632-9da3-a4af2e0e9166"
},
"example #1": {
"value": "952bea17-3776-11ea-9341-42010a84012a"
}
}
}
]
},
"/{Id}/sub1": {
"get": {
"summary": "/{Id}/sub1",
"description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.111 seconds",
"operationId": "6f168b6d-9b67-490e-9895-903b078de4c5",
"responses": {
"200": {
"description": "Successful call with status 200",
"content": {
"text/html": {}
}
}
},
"x-last-seen-ts": 1567750483.864529,
"x-counters-total": {
"entries": 1,
"failures": 0,
"firstSeen": 1567750483.864529,
"lastSeen": 1567750483.864529,
"sumRT": 0.111,
"sumDuration": 0
},
"x-counters-per-source": {
"": {
"entries": 1,
"failures": 0,
"firstSeen": 1567750483.864529,
"lastSeen": 1567750483.864529,
"sumRT": 0.111,
"sumDuration": 0
}
}
},
"parameters": [
{
"name": "Id",
"in": "path",
"required": true,
"style": "simple",
"schema": {
"type": "string"
},
"examples": {
"example #0": {
"value": "e21f7112-3d3b-4632-9da3-a4af2e0e9166"
},
"example #1": {
"value": "952bea17-3776-11ea-9341-42010a84012a"
}
}
}
]
},
"/{Id}/sub2": {
"get": {
"summary": "/{Id}/sub2",
"description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.630 seconds",
"operationId": "d1e7900f-01dc-4d79-912e-5ca79ecd73e3",
"responses": {
"200": {
"description": "Successful call with status 200",
"content": {
"application/json": {
"example": null
}
}
}
},
"x-counters-per-source": {
"": {
"entries": 1,
"failures": 0,
"firstSeen": 1567750578.7471218,
"lastSeen": 1567750578.7471218,
"sumRT": 0.63,
"sumDuration": 0
}
},
"x-last-seen-ts": 1567750578.7471218,
"x-counters-total": {
"entries": 1,
"failures": 0,
"firstSeen": 1567750578.7471218,
"lastSeen": 1567750578.7471218,
"sumRT": 0.63,
"sumDuration": 0
}
},
"parameters": [
{
"name": "Id",
"in": "path",
"required": true,
"style": "simple",
"schema": {
"type": "string"
},
"examples": {
"example #0": {
"value": "e21f7112-3d3b-4632-9da3-a4af2e0e9166"
},
"example #1": {
"value": "952bea17-3776-11ea-9341-42010a84012a"
}
}
}
]
}
},
"x-counters-total": {
"entries": 9,
"failures": 0,
"firstSeen": 1567750483.864529,
"lastSeen": 1567750582.7471218,
"sumRT": 3.2639999999999993,
"sumDuration": 2
},
"x-counters-per-source": {
"": {
"entries": 9,
"failures": 0,
"firstSeen": 1567750483.864529,
"lastSeen": 1567750582.7471218,
"sumRT": 3.2639999999999993,
"sumDuration": 2
}
}
}

63
agent/pkg/oas/utils.go
View File

@ -52,8 +52,7 @@ func createSimpleParam(name string, in openapi.In, ptype openapi.SchemaType) *op
}
required := true // FFS! https://stackoverflow.com/questions/32364027/reference-a-boolean-for-assignment-in-a-struct/32364093
schema := new(openapi.SchemaObj)
schema.Type = make(openapi.Types, 0)
schema.Type = append(schema.Type, ptype)
schema.Type = openapi.Types{ptype}
style := openapi.StyleSimple
if in == openapi.InQuery {
@ -197,7 +196,7 @@ func fillParamExample(param **openapi.Examples, exampleValue string) error {
continue
}
if value == exampleValue || cnt > 5 { // 5 examples is enough
if value == exampleValue || cnt >= 5 { // 5 examples is enough
return nil
}
}
@ -213,6 +212,36 @@ func fillParamExample(param **openapi.Examples, exampleValue string) error {
return nil
}
// TODO: somehow generalize the two example setting functions, plus add body example handling
func addSchemaExample(existing *openapi.SchemaObj, bodyStr string) {
if len(existing.Examples) < 5 {
found := false
for _, eVal := range existing.Examples {
existingExample := ""
err := json.Unmarshal(eVal, &existingExample)
if err != nil {
logger.Log.Debugf("Failed to unmarshal example: %v", eVal)
continue
}
if existingExample == bodyStr {
found = true
break
}
}
if !found {
example, err := json.Marshal(bodyStr)
if err != nil {
logger.Log.Debugf("Failed to marshal example: %v", bodyStr)
return
}
existing.Examples = append(existing.Examples, example)
}
}
}
func longestCommonXfix(strs [][]string, pre bool) []string { // https://github.com/jpillora/longestcommon
empty := make([]string, 0)
//short-circuit empty list
@ -370,3 +399,31 @@ func isAlphaRune(r rune) bool {
func isAlNumRune(b rune) bool {
return isAlphaRune(b) || ('0' <= b && b <= '9')
}
func deleteFromSlice(s []string, val string) []string {
temp := s[:0]
for _, x := range s {
if x != val {
temp = append(temp, x)
}
}
return temp
}
func sliceContains(s []string, e string) bool {
for _, a := range s {
if a == e {
return true
}
}
return false
}
func intersectSliceWithMap(required []string, names map[string]struct{}) []string {
for name := range names {
if !sliceContains(required, name) {
required = deleteFromSlice(required, name)
}
}
return required
}