From 0f77cf8ef94a3b6dace8b743e0b378274eef6387 Mon Sep 17 00:00:00 2001 From: Andrey Pokhilko Date: Mon, 7 Feb 2022 10:12:05 +0300 Subject: [PATCH] OAS: Some minor fixes (#762) * Commit some fixes * Accept expected Co-authored-by: Igor Gov --- agent/pkg/api/main.go | 2 +- agent/pkg/oas/counters.go | 4 + agent/pkg/oas/feeder_test.go | 37 ++++-- agent/pkg/oas/oas_generator.go | 1 + agent/pkg/oas/specgen.go | 29 ++++- .../oas/test_artifacts/params.har.spec.json | 115 ++++++++++-------- agent/pkg/oas/utils.go | 4 +- 7 files changed, 120 insertions(+), 72 deletions(-) diff --git a/agent/pkg/api/main.go b/agent/pkg/api/main.go index a6228ae6b..3886af658 100644 --- a/agent/pkg/api/main.go +++ b/agent/pkg/api/main.go @@ -140,7 +140,7 @@ func startReadingChannel(outputItems <-chan *tapApi.OutputChannelItem, extension mizuEntry.Rules = rules } - entryWSource := oas.EntryWithSource{Entry: *harEntry, Source: mizuEntry.Source.Name} + entryWSource := oas.EntryWithSource{Entry: *harEntry, Source: mizuEntry.Source.Name, Id: mizuEntry.Id} oas.GetOasGeneratorInstance().PushEntry(&entryWSource) } diff --git a/agent/pkg/oas/counters.go b/agent/pkg/oas/counters.go index ff7ac52d4..3a8f28fd5 100644 --- a/agent/pkg/oas/counters.go +++ b/agent/pkg/oas/counters.go @@ -17,6 +17,10 @@ type Counter struct { } func (c *Counter) addEntry(ts float64, rt float64, succ bool, dur float64) { + if dur < 0 { + panic("Duration cannot be negative") + } + c.Entries += 1 c.SumRT += rt c.SumDuration += dur diff --git a/agent/pkg/oas/feeder_test.go b/agent/pkg/oas/feeder_test.go index 59fd451e4..4e78734ce 100644 --- a/agent/pkg/oas/feeder_test.go +++ b/agent/pkg/oas/feeder_test.go @@ -21,20 +21,33 @@ func getFiles(baseDir string) (result []string, err error) { result = make([]string, 0) logger.Log.Infof("Reading files from tree: %s", baseDir) + inputs := []string{baseDir} + // https://yourbasic.org/golang/list-files-in-directory/ - err = filepath.Walk(baseDir, - func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - - ext := strings.ToLower(filepath.Ext(path)) - if !info.IsDir() && (ext == ".har" || ext == ".ldjson") { - result = append(result, path) - } + visitor := func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.Mode()&os.ModeSymlink != 0 { + path, _ = os.Readlink(path) + inputs = append(inputs, path) return nil - }) + } + + ext := strings.ToLower(filepath.Ext(path)) + if !info.IsDir() && (ext == ".har" || ext == ".ldjson") { + result = append(result, path) + } + + return nil + } + + for len(inputs) > 0 { + path := inputs[0] + inputs = inputs[1:] + err = filepath.Walk(path, visitor) + } sort.SliceStable(result, func(i, j int) bool { return fileSize(result[i]) < fileSize(result[j]) @@ -126,7 +139,7 @@ func feedEntry(entry *har.Entry, source string, isSync bool, file string) { logger.Log.Debugf("Interesting: %s", entry.Request.URL) } - ews := EntryWithSource{Entry: *entry, Source: source} + ews := EntryWithSource{Entry: *entry, Source: source, Id: uint(0)} if isSync { GetOasGeneratorInstance().entriesChan <- ews // blocking variant, right? } else { diff --git a/agent/pkg/oas/oas_generator.go b/agent/pkg/oas/oas_generator.go index 7f3aef7ed..0f1f44b3a 100644 --- a/agent/pkg/oas/oas_generator.go +++ b/agent/pkg/oas/oas_generator.go @@ -107,6 +107,7 @@ func newOasGenerator() *oasGenerator { type EntryWithSource struct { Source string Entry har.Entry + Id uint } type oasGenerator struct { diff --git a/agent/pkg/oas/specgen.go b/agent/pkg/oas/specgen.go index 36b20b11b..ec542595b 100644 --- a/agent/pkg/oas/specgen.go +++ b/agent/pkg/oas/specgen.go @@ -26,6 +26,7 @@ import ( const LastSeenTS = "x-last-seen-ts" const CountersTotal = "x-counters-total" const CountersPerSource = "x-counters-per-source" +const SampleId = "x-sample-entry" type reqResp struct { // hello, generics in Go Req *har.Request @@ -306,7 +307,7 @@ func handleCounters(opObj *openapi.Operation, success bool, entryWithSource *Ent rt := float64(entryWithSource.Entry.Time) / 1000 dur := 0.0 - if prevTs != 0 { + if prevTs != 0 && ts >= prevTs { dur = ts - prevTs } @@ -328,19 +329,39 @@ func handleCounters(opObj *openapi.Operation, success bool, entryWithSource *Ent return err } + err = opObj.Extensions.SetExtension(SampleId, entryWithSource.Id) + if err != nil { + return err + } + return nil } func handleRequest(req *har.Request, opObj *openapi.Operation, isSuccess bool) error { // TODO: we don't handle the situation when header/qstr param can be defined on pathObj level. Also the path param defined on opObj + urlParsed, err := url.Parse(req.URL) + if err != nil { + return err + } + + qs := make([]har.NVP, 0) + for name, vals := range urlParsed.Query() { + for _, val := range vals { + qs = append(qs, har.NVP{Name: name, Value: val}) + } + } + + if len(qs) != len(req.QueryString) { + logger.Log.Warningf("QStr params in HAR do not match URL: %s", req.URL) + } qstrGW := nvParams{ In: openapi.InQuery, - Pairs: req.QueryString, + Pairs: qs, IsIgnored: func(name string) bool { return false }, GeneralizeName: func(name string) string { return name }, } - handleNameVals(qstrGW, &opObj.Parameters) + handleNameVals(qstrGW, &opObj.Parameters, false) hdrGW := nvParams{ In: openapi.InHeader, @@ -348,7 +369,7 @@ func handleRequest(req *har.Request, opObj *openapi.Operation, isSuccess bool) e IsIgnored: isHeaderIgnored, GeneralizeName: strings.ToLower, } - handleNameVals(hdrGW, &opObj.Parameters) + handleNameVals(hdrGW, &opObj.Parameters, true) if isSuccess { reqBody, err := getRequestBody(req, opObj) diff --git a/agent/pkg/oas/test_artifacts/params.har.spec.json b/agent/pkg/oas/test_artifacts/params.har.spec.json index c53a4f846..3023f644c 100644 --- a/agent/pkg/oas/test_artifacts/params.har.spec.json +++ b/agent/pkg/oas/test_artifacts/params.har.spec.json @@ -2,7 +2,7 @@ "openapi": "3.1.0", "info": { "title": "https://httpbin.org", - "description": "Mizu observed 13 entries (0 failed), at 0.154 hits/s, average response time is 0.251 seconds", + "description": "Mizu observed 13 entries (0 failed), at 0.155 hits/s, average response time is 0.251 seconds", "version": "1.0" }, "servers": [ @@ -15,7 +15,7 @@ "get": { "summary": "/appears-once", "description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.630 seconds", - "operationId": "8dfc5a75-5f9b-4b57-ad88-77154a016201", + "operationId": "89aa39f6-78d0-411b-b701-a33bd77868b0", "responses": { "200": { "description": "Successful call with status 200", @@ -26,6 +26,15 @@ } } }, + "x-last-seen-ts": 1567750580.0471218, + "x-counters-total": { + "entries": 1, + "failures": 0, + "firstSeen": 1567750580.0471218, + "lastSeen": 1567750580.0471218, + "sumRT": 0.63, + "sumDuration": 0 + }, "x-counters-per-source": { "": { "entries": 1, @@ -36,22 +45,14 @@ "sumDuration": 0 } }, - "x-last-seen-ts": 1567750580.0471218, - "x-counters-total": { - "entries": 1, - "failures": 0, - "firstSeen": 1567750580.0471218, - "lastSeen": 1567750580.0471218, - "sumRT": 0.63, - "sumDuration": 0 - } + "x-sample-entry": 0 } }, "/appears-twice": { "get": { "summary": "/appears-twice", "description": "Mizu observed 2 entries (0 failed), at 0.500 hits/s, average response time is 0.630 seconds", - "operationId": "844ae464-fcee-4bd7-97cf-aaa096196c02", + "operationId": "f5e2b5a2-e01a-45f4-bde1-15a7e7a06d3c", "responses": { "200": { "description": "Successful call with status 200", @@ -62,6 +63,14 @@ } } }, + "x-counters-total": { + "entries": 2, + "failures": 0, + "firstSeen": 1567750580.7471218, + "lastSeen": 1567750581.7471218, + "sumRT": 1.26, + "sumDuration": 1 + }, "x-counters-per-source": { "": { "entries": 2, @@ -72,22 +81,15 @@ "sumDuration": 1 } }, - "x-last-seen-ts": 1567750581.7471218, - "x-counters-total": { - "entries": 2, - "failures": 0, - "firstSeen": 1567750580.7471218, - "lastSeen": 1567750581.7471218, - "sumRT": 1.26, - "sumDuration": 1 - } + "x-sample-entry": 0, + "x-last-seen-ts": 1567750581.7471218 } }, "/body-optional": { "post": { "summary": "/body-optional", - "description": "Mizu observed 3 entries (0 failed), at 0.000 hits/s, average response time is 0.001 seconds", - "operationId": "6e78e253-1f15-4ecd-9147-ba407c6fbeac", + "description": "Mizu observed 3 entries (0 failed), at 0.003 hits/s, average response time is 0.001 seconds", + "operationId": "14d5b1c2-dc03-4ee5-baaa-5c7992acc82e", "responses": { "200": { "description": "Successful call with status 200", @@ -96,15 +98,15 @@ } } }, - "x-last-seen-ts": 1567750581.757122, "x-counters-total": { "entries": 3, "failures": 0, "firstSeen": 1567750581.7471218, "lastSeen": 1567750581.757122, "sumRT": 0.003, - "sumDuration": 0 + "sumDuration": 0.010000228881835938 }, + "x-sample-entry": 0, "x-counters-per-source": { "": { "entries": 3, @@ -112,9 +114,10 @@ "firstSeen": 1567750581.7471218, "lastSeen": 1567750581.757122, "sumRT": 0.003, - "sumDuration": 0 + "sumDuration": 0.010000228881835938 } }, + "x-last-seen-ts": 1567750581.757122, "requestBody": { "description": "Generic request body", "content": { @@ -129,7 +132,7 @@ "post": { "summary": "/body-required", "description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.001 seconds", - "operationId": "1eefebc8-28c0-43d8-877f-4c5f67ff479d", + "operationId": "d0958c5a-dce6-4616-99f4-201dbc51457a", "responses": { "200": { "description": "Successful call with status 200", @@ -138,15 +141,6 @@ } } }, - "x-last-seen-ts": 1567750581.757122, - "x-counters-total": { - "entries": 1, - "failures": 0, - "firstSeen": 1567750581.757122, - "lastSeen": 1567750581.757122, - "sumRT": 0.001, - "sumDuration": 0 - }, "x-counters-per-source": { "": { "entries": 1, @@ -157,6 +151,16 @@ "sumDuration": 0 } }, + "x-sample-entry": 0, + "x-counters-total": { + "entries": 1, + "failures": 0, + "firstSeen": 1567750581.757122, + "lastSeen": 1567750581.757122, + "sumRT": 0.001, + "sumDuration": 0 + }, + "x-last-seen-ts": 1567750581.757122, "requestBody": { "description": "Generic request body", "content": { @@ -172,7 +176,7 @@ "post": { "summary": "/form-multipart", "description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.001 seconds", - "operationId": "fc1eb6a3-3d55-4b4a-bfd0-c2ae85d590d5", + "operationId": "cab5a2f3-c18a-4d5a-8f92-e40da4fd6603", "responses": { "200": { "description": "Successful call with status 200", @@ -183,6 +187,7 @@ } } }, + "x-sample-entry": 0, "x-counters-per-source": { "": { "entries": 1, @@ -239,7 +244,7 @@ "post": { "summary": "/form-urlencoded", "description": "Mizu observed 2 entries (0 failed), at 0.500 hits/s, average response time is 0.001 seconds", - "operationId": "b416c98b-4ba8-4739-bd2a-6c3e31478bc9", + "operationId": "7c373ad7-6ab5-422e-971b-1cf56b18a7a2", "responses": { "200": { "description": "Successful call with status 200", @@ -248,6 +253,7 @@ } } }, + "x-last-seen-ts": 1567750581.7471218, "x-counters-total": { "entries": 2, "failures": 0, @@ -266,7 +272,7 @@ "sumDuration": 1 } }, - "x-last-seen-ts": 1567750581.7471218, + "x-sample-entry": 0, "requestBody": { "description": "Generic request body", "content": { @@ -317,7 +323,7 @@ "get": { "summary": "/{Id}", "description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.630 seconds", - "operationId": "d8b4de17-4c0e-401c-891b-36a7cee390bf", + "operationId": "99f1d11f-29c0-48f9-8bf0-9f4b407c7c3f", "responses": { "200": { "description": "Successful call with status 200", @@ -328,6 +334,7 @@ } } }, + "x-sample-entry": 0, "x-last-seen-ts": 1567750579.7471218, "x-counters-total": { "entries": 1, @@ -372,7 +379,7 @@ "get": { "summary": "/{Id}/sub1", "description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.111 seconds", - "operationId": "edacc275-1761-4011-af9b-3a3ed7c3d2b8", + "operationId": "f7e299d2-253c-4eef-975c-9a5659a7fc50", "responses": { "200": { "description": "Successful call with status 200", @@ -381,6 +388,7 @@ } } }, + "x-last-seen-ts": 1567750483.864529, "x-counters-total": { "entries": 1, "failures": 0, @@ -399,7 +407,7 @@ "sumDuration": 0 } }, - "x-last-seen-ts": 1567750483.864529 + "x-sample-entry": 0 }, "parameters": [ { @@ -425,7 +433,7 @@ "get": { "summary": "/{Id}/sub2", "description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.630 seconds", - "operationId": "61702da5-e4e7-4d95-8071-a6208f3d19d4", + "operationId": "23a54e06-4298-4ea5-b1f0-09b0354a0598", "responses": { "200": { "description": "Successful call with status 200", @@ -454,7 +462,8 @@ "sumRT": 0.63, "sumDuration": 0 } - } + }, + "x-sample-entry": 0 }, "parameters": [ { @@ -477,6 +486,14 @@ ] } }, + "x-counters-total": { + "entries": 13, + "failures": 0, + "firstSeen": 1567750483.864529, + "lastSeen": 1567750582.7471218, + "sumRT": 3.268, + "sumDuration": 2.010000228881836 + }, "x-counters-per-source": { "": { "entries": 13, @@ -484,15 +501,7 @@ "firstSeen": 1567750483.864529, "lastSeen": 1567750582.7471218, "sumRT": 3.268, - "sumDuration": 2 + "sumDuration": 2.010000228881836 } - }, - "x-counters-total": { - "entries": 13, - "failures": 0, - "firstSeen": 1567750483.864529, - "lastSeen": 1567750582.7471218, - "sumRT": 3.268, - "sumDuration": 2 } } \ No newline at end of file diff --git a/agent/pkg/oas/utils.go b/agent/pkg/oas/utils.go index 7d95f3b14..3c47852ee 100644 --- a/agent/pkg/oas/utils.go +++ b/agent/pkg/oas/utils.go @@ -115,10 +115,10 @@ type nvParams struct { GeneralizeName func(name string) string } -func handleNameVals(gw nvParams, params **openapi.ParameterList) { +func handleNameVals(gw nvParams, params **openapi.ParameterList, checkIgnore bool) { visited := map[string]*openapi.ParameterObj{} for _, pair := range gw.Pairs { - if gw.IsIgnored(pair.Name) { + if (checkIgnore && gw.IsIgnored(pair.Name)) || pair.Name == "" { continue }