diff --git a/agent/go.mod b/agent/go.mod index 25387613d..6165a2c24 100644 --- a/agent/go.mod +++ b/agent/go.mod @@ -4,7 +4,7 @@ go 1.17 require ( github.com/antelman107/net-wait-go v0.0.0-20210623112055-cf684aebda7b - github.com/chanced/openapi v0.0.7 + github.com/chanced/openapi v0.0.8 github.com/djherbis/atime v1.1.0 github.com/elastic/go-elasticsearch/v7 v7.17.0 github.com/getkin/kin-openapi v0.89.0 diff --git a/agent/go.sum b/agent/go.sum index deac6cd20..14aa39dbd 100644 --- a/agent/go.sum +++ b/agent/go.sum @@ -132,8 +132,8 @@ github.com/chanced/cmpjson v0.0.0-20210415035445-da9262c1f20a/go.mod h1:yhcmlFk1 github.com/chanced/dynamic v0.0.0-20210502140838-c010b5fc3e44/go.mod h1:XVNfXN5kgZST4PQ0W/oBAHJku2OteCeHxjAbvfd0ARM= github.com/chanced/dynamic v0.0.0-20211210164248-f8fadb1d735b h1:nQWfVfhByCAYUjDxWNMyMtq3VZ8AGOxF7wZlDnC5cTc= github.com/chanced/dynamic v0.0.0-20211210164248-f8fadb1d735b/go.mod h1:XVNfXN5kgZST4PQ0W/oBAHJku2OteCeHxjAbvfd0ARM= -github.com/chanced/openapi v0.0.7 h1:OmOBHCg/5ViUg0gaGxXBeEFoVBE8C2pHK4BO/AiD6k8= -github.com/chanced/openapi v0.0.7/go.mod h1:SxE2VMLPw+T7Vq8nwbVVhDF2PigvRF4n5XyqsVpRJGU= +github.com/chanced/openapi v0.0.8 h1:pOqKTvZEET2odGE+kJBrAdXvgpTKFPk+XRz5NTuMvrM= +github.com/chanced/openapi v0.0.8/go.mod h1:SxE2VMLPw+T7Vq8nwbVVhDF2PigvRF4n5XyqsVpRJGU= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= diff --git a/agent/pkg/oas/specgen.go b/agent/pkg/oas/specgen.go index ec542595b..60f9cf75f 100644 --- a/agent/pkg/oas/specgen.go +++ b/agent/pkg/oas/specgen.go @@ -188,7 +188,7 @@ func (g *SpecGen) handlePathObj(entryWithSource *EntryWithSource) (string, error } if entry.Request.Method == "OPTIONS" { - logger.Log.Debugf("Dropped traffic entry due to its method: %s", urlParsed.Path) + logger.Log.Debugf("Dropped traffic entry due to its method: %s %s", entry.Request.Method, urlParsed.Path) return "", nil } diff --git a/agent/pkg/oas/specgen_test.go b/agent/pkg/oas/specgen_test.go index b3872538e..f00cbb41c 100644 --- a/agent/pkg/oas/specgen_test.go +++ b/agent/pkg/oas/specgen_test.go @@ -48,7 +48,24 @@ func TestEntries(t *testing.T) { t.FailNow() } GetOasGeneratorInstance().Start() - loadStartingOAS() + loadStartingOAS("test_artifacts/catalogue.json", "catalogue") + loadStartingOAS("test_artifacts/trcc.json", "trcc-api-service") + + go func() { + for { + time.Sleep(1 * time.Second) + GetOasGeneratorInstance().ServiceSpecs.Range(func(key, val interface{}) bool { + svc := key.(string) + t.Logf("Getting spec for %s", svc) + gen := val.(*SpecGen) + _, err := gen.GetSpec() + if err != nil { + t.Error(err) + } + return true + }) + } + }() cnt, err := feedEntries(files, true) if err != nil { @@ -154,6 +171,13 @@ func TestFileSingle(t *testing.T) { t.FailNow() } + if os.Getenv("MIZU_OAS_WRITE_FILES") != "" { + err = ioutil.WriteFile(file+".spec.json", []byte(specText), 0644) + if err != nil { + panic(err) + } + } + if len(diff) > 0 { t.Errorf("Generated spec does not match expected:\n%s", diff.String()) } @@ -175,8 +199,7 @@ func waitQueueProcessed() { } } -func loadStartingOAS() { - file := "test_artifacts/catalogue.json" +func loadStartingOAS(file string, label string) { fd, err := os.Open(file) if err != nil { panic(err) @@ -195,10 +218,10 @@ func loadStartingOAS() { panic(err) } - gen := NewGen("catalogue") + gen := NewGen(label) gen.StartFromSpec(doc) - GetOasGeneratorInstance().ServiceSpecs.Store("catalogue", gen) + GetOasGeneratorInstance().ServiceSpecs.Store(label, gen) } func TestEntriesNegative(t *testing.T) { diff --git a/agent/pkg/oas/test_artifacts/params.har b/agent/pkg/oas/test_artifacts/params.har index 226f0839f..7d3bab2a5 100644 --- a/agent/pkg/oas/test_artifacts/params.har +++ b/agent/pkg/oas/test_artifacts/params.har @@ -538,6 +538,252 @@ "wait": -1, "receive": 1 } + }, + { + "startedDateTime": "2019-09-06T06:16:22.000000+00:00", + "time": 1, + "request": { + "method": "GET", + "url": "https://httpbin.org/param-patterns/prefix-gibberish-fine/234324", + "httpVersion": "", + "cookies": [], + "headers": [ + ], + "queryString": [], + "headersSize": -1, + "bodySize": -1, + "postData": { + "mimeType": "", + "text": "" + } + }, + "response": { + "status": 200, + "statusText": "OK", + "httpVersion": "", + "cookies": [], + "headers": [ + ], + "content": { + "size": 0, + "mimeType": "", + "text": "" + }, + "redirectURL": "", + "headersSize": -1, + "bodySize": 0 + }, + "cache": {}, + "timings": { + "send": -1, + "wait": -1, + "receive": 1 + } + }, + { + "startedDateTime": "2019-09-06T06:16:22.000001+00:00", + "time": 1, + "request": { + "method": "GET", + "url": "https://httpbin.org/param-patterns/prefix-gibberish-sfdlasdfkadf87sd93284q24r/1", + "httpVersion": "", + "cookies": [], + "headers": [ + ], + "queryString": [], + "headersSize": -1, + "bodySize": -1, + "postData": { + "mimeType": "", + "text": "" + } + }, + "response": { + "status": 200, + "statusText": "OK", + "httpVersion": "", + "cookies": [], + "headers": [ + ], + "content": { + "size": 0, + "mimeType": "", + "text": "" + }, + "redirectURL": "", + "headersSize": -1, + "bodySize": 0 + }, + "cache": {}, + "timings": { + "send": -1, + "wait": -1, + "receive": 1 + } + }, + { + "startedDateTime": "2019-09-06T06:16:22.000002+00:00", + "time": 1, + "request": { + "method": "GET", + "url": "https://httpbin.org/param-patterns/prefix-gibberish-adslkfasdf89sa7dfasddafa8a98sd7kansdf/static", + "httpVersion": "", + "cookies": [], + "headers": [ + ], + "queryString": [], + "headersSize": -1, + "bodySize": -1, + "postData": { + "mimeType": "", + "text": "" + } + }, + "response": { + "status": 200, + "statusText": "OK", + "httpVersion": "", + "cookies": [], + "headers": [ + ], + "content": { + "size": 0, + "mimeType": "", + "text": "" + }, + "redirectURL": "", + "headersSize": -1, + "bodySize": 0 + }, + "cache": {}, + "timings": { + "send": -1, + "wait": -1, + "receive": 1 + } + }, + { + "startedDateTime": "2019-09-06T06:16:22.000003+00:00", + "time": 1, + "request": { + "method": "GET", + "url": "https://httpbin.org/param-patterns/prefix-gibberish-4jk5l2345h2452l4352435jlk45", + "httpVersion": "", + "cookies": [], + "headers": [ + ], + "queryString": [], + "headersSize": -1, + "bodySize": -1, + "postData": { + "mimeType": "", + "text": "" + } + }, + "response": { + "status": 200, + "statusText": "OK", + "httpVersion": "", + "cookies": [], + "headers": [ + ], + "content": { + "size": 0, + "mimeType": "", + "text": "" + }, + "redirectURL": "", + "headersSize": -1, + "bodySize": 0 + }, + "cache": {}, + "timings": { + "send": -1, + "wait": -1, + "receive": 1 + } + }, + { + "startedDateTime": "2019-09-06T06:16:22.000004+00:00", + "time": 1, + "request": { + "method": "GET", + "url": "https://httpbin.org/param-patterns/prefix-gibberish-84395h2j4k35hj243j5h2kl34h54k", + "httpVersion": "", + "cookies": [], + "headers": [ + ], + "queryString": [], + "headersSize": -1, + "bodySize": -1, + "postData": { + "mimeType": "", + "text": "" + } + }, + "response": { + "status": 200, + "statusText": "OK", + "httpVersion": "", + "cookies": [], + "headers": [ + ], + "content": { + "size": 0, + "mimeType": "", + "text": "" + }, + "redirectURL": "", + "headersSize": -1, + "bodySize": 0 + }, + "cache": {}, + "timings": { + "send": -1, + "wait": -1, + "receive": 1 + } + }, + { + "startedDateTime": "2019-09-06T06:16:22.000002+00:00", + "time": 1, + "request": { + "method": "GET", + "url": "https://httpbin.org/param-patterns/prefix-gibberish-afterwards/23421", + "httpVersion": "", + "cookies": [], + "headers": [ + ], + "queryString": [], + "headersSize": -1, + "bodySize": -1, + "postData": { + "mimeType": "", + "text": "" + } + }, + "response": { + "status": 200, + "statusText": "OK", + "httpVersion": "", + "cookies": [], + "headers": [ + ], + "content": { + "size": 0, + "mimeType": "", + "text": "" + }, + "redirectURL": "", + "headersSize": -1, + "bodySize": 0 + }, + "cache": {}, + "timings": { + "send": -1, + "wait": -1, + "receive": 1 + } } ] } diff --git a/agent/pkg/oas/test_artifacts/params.har.spec.json b/agent/pkg/oas/test_artifacts/params.har.spec.json index 3023f644c..81d0fa497 100644 --- a/agent/pkg/oas/test_artifacts/params.har.spec.json +++ b/agent/pkg/oas/test_artifacts/params.har.spec.json @@ -2,7 +2,7 @@ "openapi": "3.1.0", "info": { "title": "https://httpbin.org", - "description": "Mizu observed 13 entries (0 failed), at 0.155 hits/s, average response time is 0.251 seconds", + "description": "Mizu observed 19 entries (0 failed), at 0.10 hits/s, average response time is 0.17 seconds", "version": "1.0" }, "servers": [ @@ -14,8 +14,8 @@ "/appears-once": { "get": { "summary": "/appears-once", - "description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.630 seconds", - "operationId": "89aa39f6-78d0-411b-b701-a33bd77868b0", + "description": "Mizu observed 1 entries (0 failed), at 0.00 hits/s, average response time is 0.63 seconds", + "operationId": "", "responses": { "200": { "description": "Successful call with status 200", @@ -26,33 +26,33 @@ } } }, - "x-last-seen-ts": 1567750580.0471218, - "x-counters-total": { - "entries": 1, - "failures": 0, - "firstSeen": 1567750580.0471218, - "lastSeen": 1567750580.0471218, - "sumRT": 0.63, - "sumDuration": 0 - }, "x-counters-per-source": { "": { "entries": 1, "failures": 0, - "firstSeen": 1567750580.0471218, - "lastSeen": 1567750580.0471218, + "firstSeen": 1567750580.04, + "lastSeen": 1567750580.04, "sumRT": 0.63, "sumDuration": 0 } }, + "x-counters-total": { + "entries": 1, + "failures": 0, + "firstSeen": 1567750580.04, + "lastSeen": 1567750580.04, + "sumRT": 0.63, + "sumDuration": 0 + }, + "x-last-seen-ts": 1567750580.04, "x-sample-entry": 0 } }, "/appears-twice": { "get": { "summary": "/appears-twice", - "description": "Mizu observed 2 entries (0 failed), at 0.500 hits/s, average response time is 0.630 seconds", - "operationId": "f5e2b5a2-e01a-45f4-bde1-15a7e7a06d3c", + "description": "Mizu observed 2 entries (0 failed), at 0.50 hits/s, average response time is 0.63 seconds", + "operationId": "", "responses": { "200": { "description": "Successful call with status 200", @@ -63,33 +63,33 @@ } } }, - "x-counters-total": { - "entries": 2, - "failures": 0, - "firstSeen": 1567750580.7471218, - "lastSeen": 1567750581.7471218, - "sumRT": 1.26, - "sumDuration": 1 - }, "x-counters-per-source": { "": { "entries": 2, "failures": 0, - "firstSeen": 1567750580.7471218, - "lastSeen": 1567750581.7471218, + "firstSeen": 1567750580.74, + "lastSeen": 1567750581.74, "sumRT": 1.26, "sumDuration": 1 } }, - "x-sample-entry": 0, - "x-last-seen-ts": 1567750581.7471218 + "x-counters-total": { + "entries": 2, + "failures": 0, + "firstSeen": 1567750580.74, + "lastSeen": 1567750581.74, + "sumRT": 1.26, + "sumDuration": 1 + }, + "x-last-seen-ts": 1567750581.74, + "x-sample-entry": 0 } }, "/body-optional": { "post": { "summary": "/body-optional", - "description": "Mizu observed 3 entries (0 failed), at 0.003 hits/s, average response time is 0.001 seconds", - "operationId": "14d5b1c2-dc03-4ee5-baaa-5c7992acc82e", + "description": "Mizu observed 3 entries (0 failed), at 0.00 hits/s, average response time is 0.00 seconds", + "operationId": "", "responses": { "200": { "description": "Successful call with status 200", @@ -98,26 +98,26 @@ } } }, - "x-counters-total": { - "entries": 3, - "failures": 0, - "firstSeen": 1567750581.7471218, - "lastSeen": 1567750581.757122, - "sumRT": 0.003, - "sumDuration": 0.010000228881835938 - }, - "x-sample-entry": 0, "x-counters-per-source": { "": { "entries": 3, "failures": 0, - "firstSeen": 1567750581.7471218, - "lastSeen": 1567750581.757122, - "sumRT": 0.003, - "sumDuration": 0.010000228881835938 + "firstSeen": 1567750581.74, + "lastSeen": 1567750581.75, + "sumRT": 0.00, + "sumDuration": 0.01 } }, - "x-last-seen-ts": 1567750581.757122, + "x-counters-total": { + "entries": 3, + "failures": 0, + "firstSeen": 1567750581.74, + "lastSeen": 1567750581.75, + "sumRT": 0.00, + "sumDuration": 0.01 + }, + "x-last-seen-ts": 1567750581.75, + "x-sample-entry": 0, "requestBody": { "description": "Generic request body", "content": { @@ -131,8 +131,8 @@ "/body-required": { "post": { "summary": "/body-required", - "description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.001 seconds", - "operationId": "d0958c5a-dce6-4616-99f4-201dbc51457a", + "description": "Mizu observed 1 entries (0 failed), at 0.00 hits/s, average response time is 0.00 seconds", + "operationId": "", "responses": { "200": { "description": "Successful call with status 200", @@ -145,22 +145,22 @@ "": { "entries": 1, "failures": 0, - "firstSeen": 1567750581.757122, - "lastSeen": 1567750581.757122, - "sumRT": 0.001, + "firstSeen": 1567750581.75, + "lastSeen": 1567750581.75, + "sumRT": 0.00, "sumDuration": 0 } }, - "x-sample-entry": 0, "x-counters-total": { "entries": 1, "failures": 0, - "firstSeen": 1567750581.757122, - "lastSeen": 1567750581.757122, - "sumRT": 0.001, + "firstSeen": 1567750581.75, + "lastSeen": 1567750581.75, + "sumRT": 0.00, "sumDuration": 0 }, - "x-last-seen-ts": 1567750581.757122, + "x-last-seen-ts": 1567750581.75, + "x-sample-entry": 0, "requestBody": { "description": "Generic request body", "content": { @@ -175,8 +175,8 @@ "/form-multipart": { "post": { "summary": "/form-multipart", - "description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.001 seconds", - "operationId": "cab5a2f3-c18a-4d5a-8f92-e40da4fd6603", + "description": "Mizu observed 1 entries (0 failed), at 0.00 hits/s, average response time is 0.00 seconds", + "operationId": "", "responses": { "200": { "description": "Successful call with status 200", @@ -187,26 +187,26 @@ } } }, - "x-sample-entry": 0, "x-counters-per-source": { "": { "entries": 1, "failures": 0, - "firstSeen": 1567750582.7471218, - "lastSeen": 1567750582.7471218, - "sumRT": 0.001, + "firstSeen": 1567750582.74, + "lastSeen": 1567750582.74, + "sumRT": 0.00, "sumDuration": 0 } }, - "x-last-seen-ts": 1567750582.7471218, "x-counters-total": { "entries": 1, "failures": 0, - "firstSeen": 1567750582.7471218, - "lastSeen": 1567750582.7471218, - "sumRT": 0.001, + "firstSeen": 1567750582.74, + "lastSeen": 1567750582.74, + "sumRT": 0.00, "sumDuration": 0 }, + "x-last-seen-ts": 1567750582.74, + "x-sample-entry": 0, "requestBody": { "description": "Generic request body", "content": { @@ -243,8 +243,8 @@ "/form-urlencoded": { "post": { "summary": "/form-urlencoded", - "description": "Mizu observed 2 entries (0 failed), at 0.500 hits/s, average response time is 0.001 seconds", - "operationId": "7c373ad7-6ab5-422e-971b-1cf56b18a7a2", + "description": "Mizu observed 2 entries (0 failed), at 0.50 hits/s, average response time is 0.00 seconds", + "operationId": "", "responses": { "200": { "description": "Successful call with status 200", @@ -253,25 +253,25 @@ } } }, - "x-last-seen-ts": 1567750581.7471218, - "x-counters-total": { - "entries": 2, - "failures": 0, - "firstSeen": 1567750580.7471218, - "lastSeen": 1567750581.7471218, - "sumRT": 0.002, - "sumDuration": 1 - }, "x-counters-per-source": { "": { "entries": 2, "failures": 0, - "firstSeen": 1567750580.7471218, - "lastSeen": 1567750581.7471218, - "sumRT": 0.002, + "firstSeen": 1567750580.74, + "lastSeen": 1567750581.74, + "sumRT": 0.00, "sumDuration": 1 } }, + "x-counters-total": { + "entries": 2, + "failures": 0, + "firstSeen": 1567750580.74, + "lastSeen": 1567750581.74, + "sumRT": 0.00, + "sumDuration": 1 + }, + "x-last-seen-ts": 1567750581.74, "x-sample-entry": 0, "requestBody": { "description": "Generic request body", @@ -319,11 +319,347 @@ } } }, + "/param-patterns/prefix-gibberish-fine/{prefixgibberishfineId}": { + "get": { + "tags": [ + "param-patterns" + ], + "summary": "/param-patterns/prefix-gibberish-fine/{prefixgibberishfineId}", + "description": "Mizu observed 1 entries (0 failed), at 0.00 hits/s, average response time is 0.00 seconds", + "operationId": "", + "responses": { + "200": { + "description": "Successful call with status 200", + "content": { + "": {} + } + } + }, + "x-counters-per-source": { + "": { + "entries": 1, + "failures": 0, + "firstSeen": 1567750582, + "lastSeen": 1567750582, + "sumRT": 0.00, + "sumDuration": 0 + } + }, + "x-counters-total": { + "entries": 1, + "failures": 0, + "firstSeen": 1567750582, + "lastSeen": 1567750582, + "sumRT": 0.00, + "sumDuration": 0 + }, + "x-last-seen-ts": 1567750582, + "x-sample-entry": 0 + }, + "parameters": [ + { + "name": "prefixgibberishfineId", + "in": "path", + "required": true, + "style": "simple", + "schema": { + "type": "string" + }, + "examples": { + "example #0": { + "value": "234324" + } + } + } + ] + }, + "/param-patterns/{parampatternId}": { + "get": { + "tags": [ + "param-patterns" + ], + "summary": "/param-patterns/{parampatternId}", + "description": "Mizu observed 2 entries (0 failed), at 0.00 hits/s, average response time is 0.00 seconds", + "operationId": "", + "responses": { + "200": { + "description": "Successful call with status 200", + "content": { + "": {} + } + } + }, + "x-counters-per-source": { + "": { + "entries": 2, + "failures": 0, + "firstSeen": 1567750582.00, + "lastSeen": 1567750582.00, + "sumRT": 0.00, + "sumDuration": 9.53e-7 + } + }, + "x-counters-total": { + "entries": 2, + "failures": 0, + "firstSeen": 1567750582.00, + "lastSeen": 1567750582.00, + "sumRT": 0.00, + "sumDuration": 9.53e-7 + }, + "x-last-seen-ts": 1567750582.00, + "x-sample-entry": 0 + }, + "parameters": [ + { + "name": "parampatternId", + "in": "path", + "required": true, + "style": "simple", + "schema": { + "type": "string", + "pattern": "^prefix-gibberish-.+" + }, + "examples": { + "example #0": { + "value": "prefix-gibberish-sfdlasdfkadf87sd93284q24r" + }, + "example #1": { + "value": "prefix-gibberish-adslkfasdf89sa7dfasddafa8a98sd7kansdf" + }, + "example #2": { + "value": "prefix-gibberish-4jk5l2345h2452l4352435jlk45" + }, + "example #3": { + "value": "prefix-gibberish-84395h2j4k35hj243j5h2kl34h54k" + }, + "example #4": { + "value": "prefix-gibberish-afterwards" + } + } + } + ] + }, + "/param-patterns/{parampatternId}/1": { + "get": { + "tags": [ + "param-patterns" + ], + "summary": "/param-patterns/{parampatternId}/1", + "description": "Mizu observed 1 entries (0 failed), at 0.00 hits/s, average response time is 0.00 seconds", + "operationId": "", + "responses": { + "200": { + "description": "Successful call with status 200", + "content": { + "": {} + } + } + }, + "x-counters-per-source": { + "": { + "entries": 1, + "failures": 0, + "firstSeen": 1567750582.00, + "lastSeen": 1567750582.00, + "sumRT": 0.00, + "sumDuration": 0 + } + }, + "x-counters-total": { + "entries": 1, + "failures": 0, + "firstSeen": 1567750582.00, + "lastSeen": 1567750582.00, + "sumRT": 0.00, + "sumDuration": 0 + }, + "x-last-seen-ts": 1567750582.00, + "x-sample-entry": 0 + }, + "parameters": [ + { + "name": "parampatternId", + "in": "path", + "required": true, + "style": "simple", + "schema": { + "type": "string", + "pattern": "^prefix-gibberish-.+" + }, + "examples": { + "example #0": { + "value": "prefix-gibberish-sfdlasdfkadf87sd93284q24r" + }, + "example #1": { + "value": "prefix-gibberish-adslkfasdf89sa7dfasddafa8a98sd7kansdf" + }, + "example #2": { + "value": "prefix-gibberish-4jk5l2345h2452l4352435jlk45" + }, + "example #3": { + "value": "prefix-gibberish-84395h2j4k35hj243j5h2kl34h54k" + }, + "example #4": { + "value": "prefix-gibberish-afterwards" + } + } + } + ] + }, + "/param-patterns/{parampatternId}/static": { + "get": { + "tags": [ + "param-patterns" + ], + "summary": "/param-patterns/{parampatternId}/static", + "description": "Mizu observed 1 entries (0 failed), at 0.00 hits/s, average response time is 0.00 seconds", + "operationId": "", + "responses": { + "200": { + "description": "Successful call with status 200", + "content": { + "": {} + } + } + }, + "x-counters-per-source": { + "": { + "entries": 1, + "failures": 0, + "firstSeen": 1567750582.00, + "lastSeen": 1567750582.00, + "sumRT": 0.00, + "sumDuration": 0 + } + }, + "x-counters-total": { + "entries": 1, + "failures": 0, + "firstSeen": 1567750582.00, + "lastSeen": 1567750582.00, + "sumRT": 0.00, + "sumDuration": 0 + }, + "x-last-seen-ts": 1567750582.00, + "x-sample-entry": 0 + }, + "parameters": [ + { + "name": "parampatternId", + "in": "path", + "required": true, + "style": "simple", + "schema": { + "type": "string", + "pattern": "^prefix-gibberish-.+" + }, + "examples": { + "example #0": { + "value": "prefix-gibberish-sfdlasdfkadf87sd93284q24r" + }, + "example #1": { + "value": "prefix-gibberish-adslkfasdf89sa7dfasddafa8a98sd7kansdf" + }, + "example #2": { + "value": "prefix-gibberish-4jk5l2345h2452l4352435jlk45" + }, + "example #3": { + "value": "prefix-gibberish-84395h2j4k35hj243j5h2kl34h54k" + }, + "example #4": { + "value": "prefix-gibberish-afterwards" + } + } + } + ] + }, + "/param-patterns/{parampatternId}/{param1}": { + "get": { + "tags": [ + "param-patterns" + ], + "summary": "/param-patterns/{parampatternId}/{param1}", + "description": "Mizu observed 1 entries (0 failed), at 0.00 hits/s, average response time is 0.00 seconds", + "operationId": "", + "responses": { + "200": { + "description": "Successful call with status 200", + "content": { + "": {} + } + } + }, + "x-counters-per-source": { + "": { + "entries": 1, + "failures": 0, + "firstSeen": 1567750582.00, + "lastSeen": 1567750582.00, + "sumRT": 0.00, + "sumDuration": 0 + } + }, + "x-counters-total": { + "entries": 1, + "failures": 0, + "firstSeen": 1567750582.00, + "lastSeen": 1567750582.00, + "sumRT": 0.00, + "sumDuration": 0 + }, + "x-last-seen-ts": 1567750582.00, + "x-sample-entry": 0 + }, + "parameters": [ + { + "name": "param1", + "in": "path", + "required": true, + "style": "simple", + "schema": { + "type": "string" + }, + "examples": { + "example #0": { + "value": "23421" + } + } + }, + { + "name": "parampatternId", + "in": "path", + "required": true, + "style": "simple", + "schema": { + "type": "string", + "pattern": "^prefix-gibberish-.+" + }, + "examples": { + "example #0": { + "value": "prefix-gibberish-sfdlasdfkadf87sd93284q24r" + }, + "example #1": { + "value": "prefix-gibberish-adslkfasdf89sa7dfasddafa8a98sd7kansdf" + }, + "example #2": { + "value": "prefix-gibberish-4jk5l2345h2452l4352435jlk45" + }, + "example #3": { + "value": "prefix-gibberish-84395h2j4k35hj243j5h2kl34h54k" + }, + "example #4": { + "value": "prefix-gibberish-afterwards" + } + } + } + ] + }, "/{Id}": { "get": { "summary": "/{Id}", - "description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.630 seconds", - "operationId": "99f1d11f-29c0-48f9-8bf0-9f4b407c7c3f", + "description": "Mizu observed 1 entries (0 failed), at 0.00 hits/s, average response time is 0.63 seconds", + "operationId": "", "responses": { "200": { "description": "Successful call with status 200", @@ -334,26 +670,26 @@ } } }, - "x-sample-entry": 0, - "x-last-seen-ts": 1567750579.7471218, - "x-counters-total": { - "entries": 1, - "failures": 0, - "firstSeen": 1567750579.7471218, - "lastSeen": 1567750579.7471218, - "sumRT": 0.63, - "sumDuration": 0 - }, "x-counters-per-source": { "": { "entries": 1, "failures": 0, - "firstSeen": 1567750579.7471218, - "lastSeen": 1567750579.7471218, + "firstSeen": 1567750579.74, + "lastSeen": 1567750579.74, "sumRT": 0.63, "sumDuration": 0 } - } + }, + "x-counters-total": { + "entries": 1, + "failures": 0, + "firstSeen": 1567750579.74, + "lastSeen": 1567750579.74, + "sumRT": 0.63, + "sumDuration": 0 + }, + "x-last-seen-ts": 1567750579.74, + "x-sample-entry": 0 }, "parameters": [ { @@ -366,10 +702,10 @@ }, "examples": { "example #0": { - "value": "e21f7112-3d3b-4632-9da3-a4af2e0e9166" + "value": "" }, "example #1": { - "value": "952bea17-3776-11ea-9341-42010a84012a" + "value": "" } } } @@ -378,8 +714,8 @@ "/{Id}/sub1": { "get": { "summary": "/{Id}/sub1", - "description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.111 seconds", - "operationId": "f7e299d2-253c-4eef-975c-9a5659a7fc50", + "description": "Mizu observed 1 entries (0 failed), at 0.00 hits/s, average response time is 0.11 seconds", + "operationId": "", "responses": { "200": { "description": "Successful call with status 200", @@ -388,25 +724,25 @@ } } }, - "x-last-seen-ts": 1567750483.864529, - "x-counters-total": { - "entries": 1, - "failures": 0, - "firstSeen": 1567750483.864529, - "lastSeen": 1567750483.864529, - "sumRT": 0.111, - "sumDuration": 0 - }, "x-counters-per-source": { "": { "entries": 1, "failures": 0, - "firstSeen": 1567750483.864529, - "lastSeen": 1567750483.864529, - "sumRT": 0.111, + "firstSeen": 1567750483.86, + "lastSeen": 1567750483.86, + "sumRT": 0.11, "sumDuration": 0 } }, + "x-counters-total": { + "entries": 1, + "failures": 0, + "firstSeen": 1567750483.86, + "lastSeen": 1567750483.86, + "sumRT": 0.11, + "sumDuration": 0 + }, + "x-last-seen-ts": 1567750483.86, "x-sample-entry": 0 }, "parameters": [ @@ -420,10 +756,10 @@ }, "examples": { "example #0": { - "value": "e21f7112-3d3b-4632-9da3-a4af2e0e9166" + "value": "" }, "example #1": { - "value": "952bea17-3776-11ea-9341-42010a84012a" + "value": "" } } } @@ -432,8 +768,8 @@ "/{Id}/sub2": { "get": { "summary": "/{Id}/sub2", - "description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.630 seconds", - "operationId": "23a54e06-4298-4ea5-b1f0-09b0354a0598", + "description": "Mizu observed 1 entries (0 failed), at 0.00 hits/s, average response time is 0.63 seconds", + "operationId": "", "responses": { "200": { "description": "Successful call with status 200", @@ -444,25 +780,25 @@ } } }, - "x-last-seen-ts": 1567750578.7471218, - "x-counters-total": { - "entries": 1, - "failures": 0, - "firstSeen": 1567750578.7471218, - "lastSeen": 1567750578.7471218, - "sumRT": 0.63, - "sumDuration": 0 - }, "x-counters-per-source": { "": { "entries": 1, "failures": 0, - "firstSeen": 1567750578.7471218, - "lastSeen": 1567750578.7471218, + "firstSeen": 1567750578.74, + "lastSeen": 1567750578.74, "sumRT": 0.63, "sumDuration": 0 } }, + "x-counters-total": { + "entries": 1, + "failures": 0, + "firstSeen": 1567750578.74, + "lastSeen": 1567750578.74, + "sumRT": 0.63, + "sumDuration": 0 + }, + "x-last-seen-ts": 1567750578.74, "x-sample-entry": 0 }, "parameters": [ @@ -476,32 +812,32 @@ }, "examples": { "example #0": { - "value": "e21f7112-3d3b-4632-9da3-a4af2e0e9166" + "value": "" }, "example #1": { - "value": "952bea17-3776-11ea-9341-42010a84012a" + "value": "" } } } ] } }, - "x-counters-total": { - "entries": 13, - "failures": 0, - "firstSeen": 1567750483.864529, - "lastSeen": 1567750582.7471218, - "sumRT": 3.268, - "sumDuration": 2.010000228881836 - }, "x-counters-per-source": { "": { - "entries": 13, + "entries": 19, "failures": 0, - "firstSeen": 1567750483.864529, - "lastSeen": 1567750582.7471218, - "sumRT": 3.268, - "sumDuration": 2.010000228881836 + "firstSeen": 1567750483.86, + "lastSeen": 1567750582.74, + "sumRT": 3.27, + "sumDuration": 2.01 } + }, + "x-counters-total": { + "entries": 19, + "failures": 0, + "firstSeen": 1567750483.86, + "lastSeen": 1567750582.74, + "sumRT": 3.27, + "sumDuration": 2.01 } } \ No newline at end of file diff --git a/agent/pkg/oas/test_artifacts/trcc.json b/agent/pkg/oas/test_artifacts/trcc.json new file mode 100644 index 000000000..a79df1216 --- /dev/null +++ b/agent/pkg/oas/test_artifacts/trcc.json @@ -0,0 +1,50 @@ +{ + "openapi": "3.1.0", + "info": { + "title": "Preloaded TRCC", + "version": "0.1", + "description": "Test file for loading pre-existing OAS" + }, + "paths": { + "/models/{id}": { + "parameters": [ + { + "name": "id", + "in": "path", + "required": true, + "style": "simple", + "schema": { + "type": "string", + "pattern": ".+(_|-|\\.).+" + }, + "example": "some-uuid-maybe" + } + ] + }, + "/models/{id}/{id2}": { + "parameters": [ + { + "name": "id", + "in": "path", + "required": true, + "style": "simple", + "schema": { + "type": "string", + "pattern": ".+(_|-|\\.).+" + }, + "example": "some-uuid-maybe" + }, + { + "name": "id2", + "in": "path", + "required": true, + "style": "simple", + "schema": { + "type": "string", + "pattern": "\\d+" + } + } + ] + } + } +} diff --git a/agent/pkg/oas/tree.go b/agent/pkg/oas/tree.go index 81ed6d778..e9b73d3f4 100644 --- a/agent/pkg/oas/tree.go +++ b/agent/pkg/oas/tree.go @@ -1,12 +1,13 @@ package oas import ( - "net/url" - "strconv" - "strings" - + "encoding/json" "github.com/chanced/openapi" "github.com/up9inc/mizu/shared/logger" + "net/url" + "regexp" + "strconv" + "strings" ) type NodePath = []string @@ -49,8 +50,8 @@ func (n *Node) getOrSet(path NodePath, existingPathObj *openapi.PathObj) (node * node = n.searchInConstants(pathChunk) } - if node == nil { - node = n.searchInParams(paramObj, chunkIsGibberish) + if node == nil && pathChunk != "" { + node = n.searchInParams(paramObj, pathChunk, chunkIsGibberish) } // still no node found, should create it @@ -76,6 +77,10 @@ func (n *Node) getOrSet(path NodePath, existingPathObj *openapi.PathObj) (node * if err != nil { logger.Log.Warningf("Failed to add example to a parameter: %s", err) } + + if len(*exmp) >= 3 && node.pathParam.Schema.Pattern == nil { // is it enough to decide on 2 samples? + node.pathParam.Schema.Pattern = getPatternFromExamples(exmp) + } } // TODO: eat up trailing slash, in a smart way: node.pathObj!=nil && path[1]=="" @@ -88,6 +93,57 @@ func (n *Node) getOrSet(path NodePath, existingPathObj *openapi.PathObj) (node * return node } +func getPatternFromExamples(exmp *openapi.Examples) *openapi.Regexp { + allInts := true + strs := make([]string, 0) + for _, example := range *exmp { + exampleObj, err := example.ResolveExample(exampleResolver) + if err != nil { + continue + } + + var value string + err = json.Unmarshal(exampleObj.Value, &value) + if err != nil { + logger.Log.Warningf("Failed decoding parameter example into string: %s", err) + continue + } + strs = append(strs, value) + + if _, err := strconv.Atoi(value); err != nil { + allInts = false + } + } + + if allInts { + re := new(openapi.Regexp) + re.Regexp = regexp.MustCompile(`\d+`) + return re + } else { + prefix := longestCommonXfixStr(strs, true) + suffix := longestCommonXfixStr(strs, false) + + pat := "" + separators := "-._/:|*,+" // TODO: we could also cut prefix till the last separator + if len(prefix) > 0 && strings.Contains(separators, string(prefix[len(prefix)-1])) { + pat = "^" + regexp.QuoteMeta(prefix) + } + + pat += ".+" + + if len(suffix) > 0 && strings.Contains(separators, string(suffix[0])) { + pat += regexp.QuoteMeta(suffix) + "$" + } + + if pat != ".+" { + re := new(openapi.Regexp) + re.Regexp = regexp.MustCompile(pat) + return re + } + } + return nil +} + func (n *Node) createParam() *openapi.ParameterObj { name := "param" @@ -118,23 +174,30 @@ func (n *Node) createParam() *openapi.ParameterObj { return newParam } -func (n *Node) searchInParams(paramObj *openapi.ParameterObj, chunkIsGibberish bool) *Node { +func (n *Node) searchInParams(paramObj *openapi.ParameterObj, chunk string, chunkIsGibberish bool) *Node { // look among params - if paramObj != nil || chunkIsGibberish { - for _, subnode := range n.children { - if subnode.constant != nil { - continue - } - - // TODO: check the regex pattern of param? for exceptions etc - - if paramObj != nil { - // TODO: mergeParam(subnode.pathParam, paramObj) - return subnode - } else { - return subnode - } + for _, subnode := range n.children { + if subnode.constant != nil { + continue } + + if paramObj != nil { + // TODO: mergeParam(subnode.pathParam, paramObj) + return subnode + } else if subnode.pathParam.Schema.Pattern != nil { // it has defined param pattern, have to respect it + // TODO: and not in exceptions + if subnode.pathParam.Schema.Pattern.Match([]byte(chunk)) { + return subnode + } else if chunkIsGibberish { + // TODO: what to do if gibberish chunk does not match the pattern and not in exceptions? + return nil + } else { + return nil + } + } else if chunkIsGibberish { + return subnode + } + } return nil } diff --git a/agent/pkg/oas/utils.go b/agent/pkg/oas/utils.go index 3c47852ee..59fb0849b 100644 --- a/agent/pkg/oas/utils.go +++ b/agent/pkg/oas/utils.go @@ -290,6 +290,53 @@ func longestCommonXfix(strs [][]string, pre bool) []string { // https://github.c return xfix } +func longestCommonXfixStr(strs []string, pre bool) string { // https://github.com/jpillora/longestcommon + //short-circuit empty list + if len(strs) == 0 { + return "" + } + xfix := strs[0] + //short-circuit single-element list + if len(strs) == 1 { + return xfix + } + //compare first to rest + for _, str := range strs[1:] { + xfixl := len(xfix) + strl := len(str) + //short-circuit empty strings + if xfixl == 0 || strl == 0 { + return "" + } + //maximum possible length + maxl := xfixl + if strl < maxl { + maxl = strl + } + //compare letters + if pre { + //prefix, iterate left to right + for i := 0; i < maxl; i++ { + if xfix[i] != str[i] { + xfix = xfix[:i] + break + } + } + } else { + //suffix, iternate right to left + for i := 0; i < maxl; i++ { + xi := xfixl - i - 1 + si := strl - i - 1 + if xfix[xi] != str[si] { + xfix = xfix[xi+1:] + break + } + } + } + } + return xfix +} + func getSimilarPrefix(strs []string) string { chunked := make([][]string, 0) for _, item := range strs {