OAS: Improve URL clustering (#711)

* Evidence of problems

* Improving it

* Progressing

* Progress

* Fixes

* progress

* log is not right

* Fixing thresholds

* Almost there

* Imptovements

* Move some false negatives into TODO

* cosmetics

* linter (I disagree)

* linter (I disagree)

* fix test

Dockerfile

@@ -72,10 +72,10 @@ ARG SEM_VER=0.0.0
 WORKDIR /app/agent-build
 
 RUN go build -ldflags="-extldflags=-static -s -w \
-    -X 'mizuserver/pkg/version.GitCommitHash=${COMMIT_HASH}' \
-    -X 'mizuserver/pkg/version.Branch=${GIT_BRANCH}' \
-    -X 'mizuserver/pkg/version.BuildTimestamp=${BUILD_TIMESTAMP}' \
-    -X 'mizuserver/pkg/version.SemVer=${SEM_VER}'" -o mizuagent .
+    -X 'github.com/up9inc/mizu/agent/pkg/version.GitCommitHash=${COMMIT_HASH}' \
+    -X 'github.com/up9inc/mizu/agent/pkg/version.Branch=${GIT_BRANCH}' \
+    -X 'github.com/up9inc/mizu/agent/pkg/version.BuildTimestamp=${BUILD_TIMESTAMP}' \
+    -X 'github.com/up9inc/mizu/agent/pkg/version.SemVer=${SEM_VER}'" -o mizuagent .
 
 # Download Basenine executable, verify the sha1sum
 ADD https://github.com/up9inc/basenine/releases/download/v0.4.13/basenine_linux_${GOARCH} ./basenine_linux_${GOARCH}

acceptanceTests/cypress/integration/tests/UiTest.js

@@ -3,6 +3,7 @@ import {resizeToHugeMizu, resizeToNormalMizu} from "../testHelpers/TrafficHelper
 const greenFilterColor = 'rgb(210, 250, 210)';
 const redFilterColor = 'rgb(250, 214, 220)';
 const refreshWaitTimeout = 10000;
+const bodyJsonClass = '.hljs';
 
 it('opening mizu', function () {
     cy.visit(Cypress.env('testUrl'));
@@ -192,11 +193,12 @@ function checkFilter(filterDetails){
             if (!applyByEnter)
                 cy.get('[type="submit"]').click();
 
-            // only one entry in DOM after filtering, checking all four checks on it
+            // only one entry in DOM after filtering, checking all checks on it
             leftTextCheck(totalEntries - 1, leftSidePath, leftSideExpectedText);
             leftOnHoverCheck(totalEntries - 1, leftSidePath, name);
             rightTextCheck(rightSidePath, rightSideExpectedText);
             rightOnHoverCheck(rightSidePath, name);
+            checkRightSideResponseBody();
 
             cy.get('[title="Fetch old records"]').click();
             resizeToHugeMizu();
@@ -227,7 +229,7 @@ function deeperChcek(leftSidePath, rightSidePath, filterName, leftSideExpectedTe
         cy.get(`#list #entry-${entryNum}`).click();
         rightTextCheck(rightSidePath, rightSideExpectedText);
         rightOnHoverCheck(rightSidePath, filterName);
-        checkRightSide();
+        checkRightSideResponseBody();
     });
 }
 
@@ -249,13 +251,12 @@ function rightOnHoverCheck(path, expectedText) {
     cy.get(`.TrafficPage-Container > :nth-child(2) .Queryable-Tooltip`).should('have.text', expectedText);
 }
 
-function checkRightSide() {
-    const jsonClass = '.hljs';
 
+function checkRightSideResponseBody() {
     cy.contains('Response').click();
     clickCheckbox('Decode Base64');
 
-    cy.get(`${jsonClass}`).then(value => {
+    cy.get(`${bodyJsonClass}`).then(value => {
         const encodedBody = value.text();
         cy.log(encodedBody);
 
@@ -264,25 +265,25 @@ function checkRightSide() {
 
         const expectdJsonBody = {
             args: RegExp({}),
-            url: RegExp("http://.*/get"),
+            url: RegExp('http://.*/get'),
             headers: {
                 "User-Agent": RegExp('[REDACTED]'),
                 "Accept-Encoding": RegExp('gzip'),
-                "X-Forwarded-Uri": RegExp("/api/v1/namespaces/.*/services/.*/proxy/get")
+                "X-Forwarded-Uri": RegExp('/api/v1/namespaces/.*/services/.*/proxy/get')
             }
-        }
+        };
 
         expect(responseBody.args).to.match(expectdJsonBody.args);
         expect(responseBody.url).to.match(expectdJsonBody.url);
-        expect(responseBody.headers["User-Agent"]).to.match(expectdJsonBody.headers["User-Agent"]);
-        expect(responseBody.headers["Accept-Encoding"]).to.match(expectdJsonBody.headers["Accept-Encoding"]);
-        expect(responseBody.headers["X-Forwarded-Uri"]).to.match(expectdJsonBody.headers["X-Forwarded-Uri"]);
+        expect(responseBody.headers['User-Agent']).to.match(expectdJsonBody.headers['User-Agent']);
+        expect(responseBody.headers['Accept-Encoding']).to.match(expectdJsonBody.headers['Accept-Encoding']);
+        expect(responseBody.headers['X-Forwarded-Uri']).to.match(expectdJsonBody.headers['X-Forwarded-Uri']);
 
-        cy.get('.hljs').should('have.text', encodedBody);
+        cy.get(`${bodyJsonClass}`).should('have.text', encodedBody);
         clickCheckbox('Decode Base64');
 
-        cy.get('.hljs > ').its('length').should('be.gt', 1).then(linesNum => {
-            cy.get('.hljs > >').its('length').should('be.gt', linesNum).then(jsonItemsNum => {
+        cy.get(`${bodyJsonClass} > `).its('length').should('be.gt', 1).then(linesNum => {
+            cy.get(`${bodyJsonClass} > >`).its('length').should('be.gt', linesNum).then(jsonItemsNum => {
                 checkPrettyAndLineNums(jsonItemsNum, decodedBody);
 
                 clickCheckbox('Line numbers');
@@ -304,7 +305,7 @@ function clickCheckbox(type) {
 
 function checkPrettyAndLineNums(jsonItemsLen, decodedBody) {
     decodedBody = decodedBody.replaceAll(' ', '');
-    cy.get('.hljs >').then(elements => {
+    cy.get(`${bodyJsonClass} >`).then(elements => {
         const lines = Object.values(elements);
         lines.forEach((line, index) => {
             if (line.getAttribute) {
@@ -324,13 +325,13 @@ function getCleanLine(lineElement) {
 }
 
 function checkPrettyOrNothing(jsonItems, decodedBody) {
-    cy.get('.hljs > ').should('have.length', jsonItems).then(text => {
+    cy.get(`${bodyJsonClass} > `).should('have.length', jsonItems).then(text => {
         const json = text.text();
         expect(json).to.equal(decodedBody);
     });
 }
 
 function checkOnlyLineNumberes(jsonItems, decodedText) {
-    cy.get('.hljs >').should('have.length', 1).and('have.text', decodedText);
-    cy.get('.hljs > >').should('have.length', jsonItems)
+    cy.get(`${bodyJsonClass} >`).should('have.length', 1).and('have.text', decodedText);
+    cy.get(`${bodyJsonClass} > >`).should('have.length', jsonItems)
 }

agent/go.mod

@@ -1,4 +1,4 @@
-module mizuserver
+module github.com/up9inc/mizu/agent
 
 go 1.16
 
@@ -15,6 +15,7 @@ require (
 	github.com/go-playground/validator/v10 v10.5.0
 	github.com/google/uuid v1.1.2
 	github.com/gorilla/websocket v1.4.2
+	github.com/nav-inc/datetime v0.1.3
 	github.com/op/go-logging v0.0.0-20160315200505-970db520ece7
 	github.com/orcaman/concurrent-map v0.0.0-20210106121528-16402b402231
 	github.com/ory/keto-client-go v0.7.0-alpha.1

agent/go.sum

@@ -515,6 +515,8 @@ github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8m
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus=
 github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
+github.com/nav-inc/datetime v0.1.3 h1:PaybPUsScX+Cd3TEa1tYpfwU61deCEhMTlCO2hONm1c=
+github.com/nav-inc/datetime v0.1.3/go.mod h1:gKGf5G+cW7qkTo5TC/sieNyz6lYdrA9cf1PNV+pXIOE=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
 github.com/ohler55/ojg v1.12.12 h1:hepbQFn7GHAecTPmwS3j5dCiOLsOpzPLvhiqnlAVAoE=
@@ -612,6 +614,7 @@ github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5q
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
+github.com/stretchr/testify v1.2.1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=

agent/main.go

@@ -6,17 +6,6 @@ import (
 	"flag"
 	"fmt"
 	"io/ioutil"
-	"mizuserver/pkg/api"
-	"mizuserver/pkg/config"
-	"mizuserver/pkg/controllers"
-	"mizuserver/pkg/elastic"
-	"mizuserver/pkg/middlewares"
-	"mizuserver/pkg/models"
-	"mizuserver/pkg/oas"
-	"mizuserver/pkg/routes"
-	"mizuserver/pkg/servicemap"
-	"mizuserver/pkg/up9"
-	"mizuserver/pkg/utils"
 	"net/http"
 	"os"
 	"os/signal"
@@ -26,6 +15,21 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/up9inc/mizu/agent/pkg/middlewares"
+	"github.com/up9inc/mizu/agent/pkg/models"
+	"github.com/up9inc/mizu/agent/pkg/oas"
+	"github.com/up9inc/mizu/agent/pkg/routes"
+	"github.com/up9inc/mizu/agent/pkg/servicemap"
+	"github.com/up9inc/mizu/agent/pkg/up9"
+	"github.com/up9inc/mizu/agent/pkg/utils"
+
+	"github.com/up9inc/mizu/agent/pkg/elastic"
+
+	"github.com/up9inc/mizu/agent/pkg/controllers"
+
+	"github.com/up9inc/mizu/agent/pkg/api"
+	"github.com/up9inc/mizu/agent/pkg/config"
+
 	v1 "k8s.io/api/core/v1"
 
 	"github.com/antelman107/net-wait-go/wait"

agent/pkg/api/main.go

@@ -5,22 +5,23 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"mizuserver/pkg/elastic"
-	"mizuserver/pkg/har"
-	"mizuserver/pkg/holder"
-	"mizuserver/pkg/providers"
 	"os"
 	"path"
 	"sort"
 	"strings"
 	"time"
 
-	"mizuserver/pkg/servicemap"
+	"github.com/up9inc/mizu/agent/pkg/elastic"
+	"github.com/up9inc/mizu/agent/pkg/har"
+	"github.com/up9inc/mizu/agent/pkg/holder"
+	"github.com/up9inc/mizu/agent/pkg/providers"
 
-	"mizuserver/pkg/models"
-	"mizuserver/pkg/oas"
-	"mizuserver/pkg/resolver"
-	"mizuserver/pkg/utils"
+	"github.com/up9inc/mizu/agent/pkg/servicemap"
+
+	"github.com/up9inc/mizu/agent/pkg/models"
+	"github.com/up9inc/mizu/agent/pkg/oas"
+	"github.com/up9inc/mizu/agent/pkg/resolver"
+	"github.com/up9inc/mizu/agent/pkg/utils"
 
 	"github.com/up9inc/mizu/shared"
 	"github.com/up9inc/mizu/shared/logger"
@@ -139,7 +140,8 @@ func startReadingChannel(outputItems <-chan *tapApi.OutputChannelItem, extension
 				mizuEntry.Rules = rules
 			}
 
-			oas.GetOasGeneratorInstance().PushEntry(harEntry)
+			entryWSource := oas.EntryWithSource{Entry: *harEntry, Source: mizuEntry.Source.Name}
+			oas.GetOasGeneratorInstance().PushEntry(&entryWSource)
 		}
 
 		data, err := json.Marshal(mizuEntry)

agent/pkg/api/socket_routes.go

@@ -3,11 +3,12 @@ package api
 import (
 	"encoding/json"
 	"fmt"
-	"mizuserver/pkg/models"
 	"net/http"
 	"sync"
 	"time"
 
+	"github.com/up9inc/mizu/agent/pkg/models"
+
 	"github.com/gin-gonic/gin"
 	"github.com/gorilla/websocket"
 	basenine "github.com/up9inc/basenine/client/go"

agent/pkg/api/socket_server_handlers.go

@@ -3,12 +3,13 @@ package api
 import (
 	"encoding/json"
 	"fmt"
-	"mizuserver/pkg/models"
-	"mizuserver/pkg/providers"
-	"mizuserver/pkg/providers/tappers"
-	"mizuserver/pkg/up9"
 	"sync"
 
+	"github.com/up9inc/mizu/agent/pkg/models"
+	"github.com/up9inc/mizu/agent/pkg/providers"
+	"github.com/up9inc/mizu/agent/pkg/providers/tappers"
+	"github.com/up9inc/mizu/agent/pkg/up9"
+
 	tapApi "github.com/up9inc/mizu/tap/api"
 
 	"github.com/up9inc/mizu/shared"

agent/pkg/controllers/config_controller.go

@@ -2,21 +2,22 @@ package controllers
 
 import (
 	"context"
+	"net/http"
+	"regexp"
+	"time"
+
 	"github.com/gin-gonic/gin"
+	"github.com/up9inc/mizu/agent/pkg/config"
+	"github.com/up9inc/mizu/agent/pkg/models"
+	"github.com/up9inc/mizu/agent/pkg/providers"
+	"github.com/up9inc/mizu/agent/pkg/providers/tapConfig"
+	"github.com/up9inc/mizu/agent/pkg/providers/tappedPods"
+	"github.com/up9inc/mizu/agent/pkg/providers/tappers"
 	"github.com/up9inc/mizu/shared"
 	"github.com/up9inc/mizu/shared/kubernetes"
 	"github.com/up9inc/mizu/shared/logger"
 	tapApi "github.com/up9inc/mizu/tap/api"
 	v1 "k8s.io/api/core/v1"
-	"mizuserver/pkg/config"
-	"mizuserver/pkg/models"
-	"mizuserver/pkg/providers"
-	"mizuserver/pkg/providers/tapConfig"
-	"mizuserver/pkg/providers/tappedPods"
-	"mizuserver/pkg/providers/tappers"
-	"net/http"
-	"regexp"
-	"time"
 )
 
 var cancelTapperSyncer context.CancelFunc

agent/pkg/controllers/entries_controller.go

@@ -2,13 +2,14 @@ package controllers
 
 import (
 	"encoding/json"
-	"mizuserver/pkg/har"
-	"mizuserver/pkg/models"
-	"mizuserver/pkg/validation"
 	"net/http"
 	"strconv"
 	"time"
 
+	"github.com/up9inc/mizu/agent/pkg/har"
+	"github.com/up9inc/mizu/agent/pkg/models"
+	"github.com/up9inc/mizu/agent/pkg/validation"
+
 	"github.com/gin-gonic/gin"
 
 	basenine "github.com/up9inc/basenine/client/go"

agent/pkg/controllers/install_controller.go

@@ -1,9 +1,10 @@
 package controllers
 
 import (
-	"mizuserver/pkg/providers"
 	"net/http"
 
+	"github.com/up9inc/mizu/agent/pkg/providers"
+
 	"github.com/gin-gonic/gin"
 	"github.com/up9inc/mizu/shared/logger"
 )

agent/pkg/controllers/metadata_controller.go

@@ -1,10 +1,11 @@
 package controllers
 
 import (
-	"github.com/gin-gonic/gin"
-	"github.com/up9inc/mizu/shared"
-	"mizuserver/pkg/version"
 	"net/http"
+
+	"github.com/gin-gonic/gin"
+	"github.com/up9inc/mizu/agent/pkg/version"
+	"github.com/up9inc/mizu/shared"
 )
 
 func GetVersion(c *gin.Context) {

agent/pkg/controllers/oas_controller.go

@@ -1,11 +1,12 @@
 package controllers
 
 import (
+	"net/http"
+
 	"github.com/chanced/openapi"
 	"github.com/gin-gonic/gin"
+	"github.com/up9inc/mizu/agent/pkg/oas"
 	"github.com/up9inc/mizu/shared/logger"
-	"mizuserver/pkg/oas"
-	"net/http"
 )
 
 func GetOASServers(c *gin.Context) {

agent/pkg/controllers/oas_controller_test.go

@@ -1,10 +1,11 @@
 package controllers
 
 import (
-	"mizuserver/pkg/oas"
 	"net/http/httptest"
 	"testing"
 
+	"github.com/up9inc/mizu/agent/pkg/oas"
+
 	"github.com/gin-gonic/gin"
 )
 

agent/pkg/controllers/service_map_controller.go

@@ -1,9 +1,10 @@
 package controllers
 
 import (
-	"mizuserver/pkg/servicemap"
 	"net/http"
 
+	"github.com/up9inc/mizu/agent/pkg/servicemap"
+
 	"github.com/gin-gonic/gin"
 )
 

agent/pkg/controllers/service_map_controller_test.go

@@ -7,7 +7,7 @@ import (
 	"net/http/httptest"
 	"testing"
 
-	"mizuserver/pkg/servicemap"
+	"github.com/up9inc/mizu/agent/pkg/servicemap"
 
 	"github.com/gin-gonic/gin"
 	"github.com/stretchr/testify/suite"

agent/pkg/controllers/status_controller.go

@@ -2,17 +2,18 @@ package controllers
 
 import (
 	"encoding/json"
+	"net/http"
+
 	"github.com/gin-gonic/gin"
+	"github.com/up9inc/mizu/agent/pkg/api"
+	"github.com/up9inc/mizu/agent/pkg/holder"
+	"github.com/up9inc/mizu/agent/pkg/providers"
+	"github.com/up9inc/mizu/agent/pkg/providers/tappedPods"
+	"github.com/up9inc/mizu/agent/pkg/providers/tappers"
+	"github.com/up9inc/mizu/agent/pkg/up9"
+	"github.com/up9inc/mizu/agent/pkg/validation"
 	"github.com/up9inc/mizu/shared"
 	"github.com/up9inc/mizu/shared/logger"
-	"mizuserver/pkg/api"
-	"mizuserver/pkg/holder"
-	"mizuserver/pkg/providers"
-	"mizuserver/pkg/providers/tappedPods"
-	"mizuserver/pkg/providers/tappers"
-	"mizuserver/pkg/up9"
-	"mizuserver/pkg/validation"
-	"net/http"
 )
 
 func HealthCheck(c *gin.Context) {

agent/pkg/controllers/user_controller.go

@@ -1,7 +1,7 @@
 package controllers
 
 import (
-	"mizuserver/pkg/providers"
+	"github.com/up9inc/mizu/agent/pkg/providers"
 
 	"github.com/gin-gonic/gin"
 	"github.com/up9inc/mizu/shared/logger"

agent/pkg/holder/main.go

@@ -1,6 +1,6 @@
 package holder
 
-import "mizuserver/pkg/resolver"
+import "github.com/up9inc/mizu/agent/pkg/resolver"
 
 var k8sResolver *resolver.Resolver
 
@@ -11,4 +11,3 @@ func SetResolver(param *resolver.Resolver) {
 func GetResolver() *resolver.Resolver {
 	return k8sResolver
 }
-

agent/pkg/middlewares/auth.go

@@ -1,8 +1,8 @@
 package middlewares
 
 import (
-	"mizuserver/pkg/config"
-	"mizuserver/pkg/providers"
+	"github.com/up9inc/mizu/agent/pkg/config"
+	"github.com/up9inc/mizu/agent/pkg/providers"
 
 	"github.com/gin-gonic/gin"
 	ory "github.com/ory/kratos-client-go"

agent/pkg/models/models.go

@@ -2,9 +2,10 @@ package models
 
 import (
 	"encoding/json"
+
+	"github.com/up9inc/mizu/agent/pkg/har"
+	"github.com/up9inc/mizu/agent/pkg/rules"
 	tapApi "github.com/up9inc/mizu/tap/api"
-	"mizuserver/pkg/har"
-	"mizuserver/pkg/rules"
 
 	basenine "github.com/up9inc/basenine/client/go"
 	"github.com/up9inc/mizu/shared"

agent/pkg/oas/counters.go

@@ -0,0 +1,57 @@
+package oas
+
+import "math"
+
+type Counter struct {
+	Entries     int     `json:"entries"`
+	Failures    int     `json:"failures"`
+	FirstSeen   float64 `json:"firstSeen"`
+	LastSeen    float64 `json:"lastSeen"`
+	SumRT       float64 `json:"sumRT"`
+	SumDuration float64 `json:"sumDuration"`
+}
+
+func (c *Counter) addEntry(ts float64, rt float64, succ bool, dur float64) {
+	c.Entries += 1
+	c.SumRT += rt
+	c.SumDuration += dur
+	if !succ {
+		c.Failures += 1
+	}
+
+	if c.FirstSeen == 0 {
+		c.FirstSeen = ts
+	} else {
+		c.FirstSeen = math.Min(c.FirstSeen, ts)
+	}
+
+	c.LastSeen = math.Max(c.LastSeen, ts)
+}
+
+func (c *Counter) addOther(other *Counter) {
+	c.Entries += other.Entries
+	c.SumRT += other.SumRT
+	c.Failures += other.Failures
+	c.SumDuration += other.SumDuration
+
+	if c.FirstSeen == 0 {
+		c.FirstSeen = other.FirstSeen
+	} else {
+		c.FirstSeen = math.Min(c.FirstSeen, other.FirstSeen)
+	}
+
+	c.LastSeen = math.Max(c.LastSeen, other.LastSeen)
+}
+
+type CounterMap map[string]*Counter
+
+func (m *CounterMap) addOther(other *CounterMap) {
+	for src, cmap := range *other {
+		if existing, ok := (*m)[src]; ok {
+			existing.addOther(cmap)
+		} else {
+			copied := *cmap
+			(*m)[src] = &copied
+		}
+	}
+}

agent/pkg/oas/feeder_test.go

@@ -6,13 +6,14 @@ import (
 	"errors"
 	"io"
 	"io/ioutil"
-	"mizuserver/pkg/har"
 	"os"
 	"path/filepath"
 	"sort"
 	"strings"
 	"testing"
 
+	"github.com/up9inc/mizu/agent/pkg/har"
+
 	"github.com/up9inc/mizu/shared/logger"
 )
 
@@ -109,25 +110,26 @@ func feedFromHAR(file string, isSync bool) (int, error) {
 	cnt := 0
 	for _, entry := range harDoc.Log.Entries {
 		cnt += 1
-		feedEntry(&entry, isSync)
+		feedEntry(&entry, "", isSync)
 	}
 
 	return cnt, nil
 }
 
-func feedEntry(entry *har.Entry, isSync bool) {
+func feedEntry(entry *har.Entry, source string, isSync bool) {
 	if entry.Response.Status == 302 {
 		logger.Log.Debugf("Dropped traffic entry due to permanent redirect status: %s", entry.StartedDateTime)
 	}
 
-	if strings.Contains(entry.Request.URL, "taboola") {
+	if strings.Contains(entry.Request.URL, "some") { // for debugging
 		logger.Log.Debugf("Interesting: %s", entry.Request.URL)
 	}
 
+	ews := EntryWithSource{Entry: *entry, Source: source}
 	if isSync {
-		GetOasGeneratorInstance().entriesChan <- *entry // blocking variant, right?
+		GetOasGeneratorInstance().entriesChan <- ews // blocking variant, right?
 	} else {
-		GetOasGeneratorInstance().PushEntry(entry)
+		GetOasGeneratorInstance().PushEntry(&ews)
 	}
 }
 
@@ -144,6 +146,7 @@ func feedFromLDJSON(file string, isSync bool) (int, error) {
 	var meta map[string]interface{}
 	buf := strings.Builder{}
 	cnt := 0
+	source := ""
 	for {
 		substr, isPrefix, err := reader.ReadLine()
 		if err == io.EOF {
@@ -163,6 +166,9 @@ func feedFromLDJSON(file string, isSync bool) (int, error) {
 			if err != nil {
 				return 0, err
 			}
+			if s, ok := meta["_source"]; ok && s != nil {
+				source = s.(string)
+			}
 		} else {
 			var entry har.Entry
 			err := json.Unmarshal([]byte(line), &entry)
@@ -170,7 +176,7 @@ func feedFromLDJSON(file string, isSync bool) (int, error) {
 				logger.Log.Warningf("Failed decoding entry: %s", line)
 			} else {
 				cnt += 1
-				feedEntry(&entry, isSync)
+				feedEntry(&entry, source, isSync)
 			}
 		}
 	}

agent/pkg/oas/gibberish.go

@@ -1,22 +1,26 @@
 package oas
 
 import (
+	"math"
 	"regexp"
 	"strings"
 	"unicode"
 )
 
 var (
-	patBase64   = regexp.MustCompile(`^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$`)
 	patUuid4    = regexp.MustCompile(`(?i)[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}`)
 	patEmail    = regexp.MustCompile(`^\w+([-+.']\w+)*@\w+([-.]\w+)*\.\w+([-.]\w+)*$`)
-	patHexLower = regexp.MustCompile(`(0x)?[0-9a-f]{6,}`)
-	patHexUpper = regexp.MustCompile(`(0x)?[0-9A-F]{6,}`)
-	patLongNum  = regexp.MustCompile(`\d{6,}`)
+	patLongNum  = regexp.MustCompile(`^\d{3,}$`)
+	patLongNumB = regexp.MustCompile(`[^\d]\d{3,}`)
+	patLongNumA = regexp.MustCompile(`\d{3,}[^\d]`)
 )
 
 func IsGibberish(str string) bool {
-	if patBase64.MatchString(str) && len(str) > 32 {
+	if IsVersionString(str) {
+		return false
+	}
+
+	if patEmail.MatchString(str) {
 		return true
 	}
 
@@ -24,16 +28,44 @@ func IsGibberish(str string) bool {
 		return true
 	}
 
-	if patEmail.MatchString(str) {
+	if patLongNum.MatchString(str) || patLongNumB.MatchString(str) || patLongNumA.MatchString(str) {
 		return true
 	}
 
-	if patHexLower.MatchString(str) || patHexUpper.MatchString(str) || patLongNum.MatchString(str) {
-		return true
+	//alNum := cleanStr(str, isAlNumRune)
+	//alpha := cleanStr(str, isAlphaRune)
+	// noiseAll := isNoisy(alNum)
+	//triAll := isTrigramBad(strings.ToLower(alpha))
+	// _ = noiseAll
+
+	isNotAlNum := func(r rune) bool { return !isAlNumRune(r) }
+	chunks := strings.FieldsFunc(str, isNotAlNum)
+	noisyLen := 0
+	alnumLen := 0
+	for _, chunk := range chunks {
+		alnumLen += len(chunk)
+		noise := isNoisy(chunk)
+		tri := isTrigramBad(strings.ToLower(chunk))
+		if noise || tri {
+			noisyLen += len(chunk)
+		}
 	}
 
-	noise := noiseLevel(str)
-	return noise >= 0.2
+	return float64(noisyLen) > 0
+
+	//if float64(noisyLen) > 0 {
+	//	return true
+	//}
+
+	//if len(chunks) > 0 && float64(noisyLen) >= float64(alnumLen)/3.0 {
+	//	return true
+	//}
+
+	//if triAll {
+	//return true
+	//}
+
+	// return false
 }
 
 func noiseLevel(str string) (score float64) {
@@ -51,21 +83,21 @@ func noiseLevel(str string) (score float64) {
 
 			// upper =>
 			case unicode.IsUpper(prev) && unicode.IsLower(char):
-				score += 0.25
+				score += 0.10
 			case unicode.IsUpper(prev) && unicode.IsDigit(char):
-				score += 0.25
+				score += 0.5
 
 			// lower =>
 			case unicode.IsLower(prev) && unicode.IsUpper(char):
 				score += 0.75
 			case unicode.IsLower(prev) && unicode.IsDigit(char):
-				score += 0.25
+				score += 0.5
 
 			// digit =>
 			case unicode.IsDigit(prev) && unicode.IsUpper(char):
 				score += 0.75
 			case unicode.IsDigit(prev) && unicode.IsLower(char):
-				score += 0.75
+				score += 1.0
 
 			// the rest is 100% noise
 			default:
@@ -75,8 +107,6 @@ func noiseLevel(str string) (score float64) {
 		prev = char
 	}
 
-	score /= cnt // weigh it
-
 	return score
 }
 
@@ -97,9 +127,59 @@ func IsVersionString(component string) bool {
 		}
 	}
 
-	if !hasV && strings.Contains(component, ".") {
+	if !hasV && !strings.Contains(component, ".") {
 		return false
 	}
 
 	return true
 }
+
+func trigramScore(str string) (float64, int) {
+	tgScore := 0.0
+	trigrams := ngrams(str, 3)
+	if len(trigrams) > 0 {
+		for _, trigram := range trigrams {
+			score, found := corpus_trigrams[trigram]
+			if found {
+				tgScore += score
+			}
+		}
+	}
+
+	return tgScore, len(trigrams)
+}
+
+func isTrigramBad(s string) bool {
+	tgScore, cnt := trigramScore(s)
+
+	if cnt > 0 {
+		val := math.Sqrt(tgScore) / float64(cnt)
+		val2 := tgScore / float64(cnt)
+		threshold := 0.005
+		bad := val < threshold
+		threshold2 := math.Log(float64(cnt)-2) * 0.1
+		bad2 := val2 < threshold2
+		return bad && bad2 // TODO: improve this logic to be clearer
+	}
+	return false
+}
+
+func isNoisy(s string) bool {
+	noise := noiseLevel(s)
+
+	if len(s) > 0 {
+		val := (noise * noise) / float64(len(s))
+		threshold := 0.6
+		bad := val > threshold
+		return bad
+	}
+	return false
+}
+
+func ngrams(s string, n int) []string {
+	result := make([]string, 0)
+	for i := 0; i < len(s)-n+1; i++ {
+		result = append(result, s[i:i+n])
+	}
+	return result
+}

agent/pkg/oas/gibberish_test.go

@@ -1,17 +1,95 @@
 package oas
 
-import "testing"
+import (
+	"testing"
+)
 
 func TestNegative(t *testing.T) {
 	cases := []string{
 		"",
+		"{}",
+		"0.0.29",
+		"0.1",
+		"1.0",
+		"1.0.0",
+		"2.1.73",
+		"abTestV2",
+		"actionText,setName,setAttribute,save,ignore,onEnd,getContext,end,get",
+		"AddUserGroupLink",
+		"advert-management.adBlockerMessage.html",
+		"agents.author.1.json",
+		"animated-gif",
 		"b", // can be valid hexadecimal
-		"GetUniversalVariableUser",
+		"big-danger-coronavirus-panic-greater-crisis",
+		"breakout-box",
 		"callback",
-		"runs",
-		"tcfv2",
-		"StartUpCheckout",
+		"core.algorithm_execution.view",
+		"core.devices.view",
+		"data.json",
+		"dialog.overlay.infinity.json",
+		"domain-input",
+		"embeddable", // lul, it's a valid HEX!
+		"embeddable_blip",
+		"E PLURIBUS UNUM",
+		"etc",
+		"eu-central-1a",
+		"fcgi-bin",
+		"footer.include.html",
+		"fullHashes:find",
+		"generate-feed",
+		"GetAds",
 		"GetCart",
+		"GetUniversalVariableUser",
+		"github-audit-exports",
+		"g.js",
+		"g.pixel",
+		".html",
+		"Hugo Michiels",
+		"image.sbix",
+		"index.html",
+		"iPad",
+		"Joanna Mazewski",
+		"LibGit2Sharp",
+		"Michael_Vaughan1.png",
+		"New RSS feed has been generated",
+		"nick-clegg",
+		"opt-out",
+		"pixel_details.html",
+		"post.json",
+		"profile-method-info",
+		"project-id",
+		"publisha.1.json",
+		"publish_and_moderate",
+		"Ronna McDaniel",
+		"rtb-h",
+		"runs",
+		"sign-up",
+		"some-uuid-maybe",
+		"stable-4.0-version.json",
+		"StartUpCheckout",
+		"Steve Flunk",
+		"sync_a9",
+		"Ted Cruz",
+		"test.png",
+		"token",
+		"ToList",
+		"v2.1.3",
+		"VersionCheck.php",
+		"v Rusiji",
+		"Walnut St",
+		"web_widget",
+		"zoom_in.cur",
+		"xray",
+		"web",
+		"vipbets1",
+		"trcc",
+		"fbpixel",
+
+		// TODO below
+		// "tcfv2",
+		// "Matt-cartoon-255x206px-small.png",
+		// "TheTelegraph_portal_white-320-small.png",
+		// "testdata-10kB.js",
 	}
 
 	for _, str := range cases {
@@ -23,44 +101,92 @@ func TestNegative(t *testing.T) {
 
 func TestPositive(t *testing.T) {
 	cases := []string{
-		"e21f7112-3d3b-4632-9da3-a4af2e0e9166",
-		"952bea17-3776-11ea-9341-42010a84012a",
-		"456795af-b48f-4a8d-9b37-3e932622c2f0",
 		"0a0d0174-b338-4520-a1c3-24f7e3d5ec50.html",
-		"6120c057c7a97b03f6986f1b",
+		"1024807212418223",
+		"11ca096cbc224a67360493d44a9903",
+		"1553183382779",
+		"1554507871",
+		"19180481",
+		"203ef0f713abcebd8d62c35c0e3f12f87d71e5e4",
+		"456795af-b48f-4a8d-9b37-3e932622c2f0",
+		"601a2bdcc5b69137248ddbbf",
+		"60fe9aaeaefe2400012df94f",
 		"610bc3fd5a77a7fa25033fb0",
 		"610bd0315a77a7fa25034368",
 		"610bd0315a77a7fa25034368zh",
+		"6120c057c7a97b03f6986f1b",
 		"710a462e",
-		"1554507871",
-		"qwerqwerasdfqwer@protonmai.com",
-		"john.dow.1981@protonmail.com",
-		"ci12NC01YzkyNTEzYzllMDRhLTAtYy5tb25pdG9yaW5nLmpzb24=", // long base64
-		"11ca096cbc224a67360493d44a9903",
-		"c738338322370b47a79251f7510dd", // prefixed hex
-		"QgAAAC6zw0qH2DJtnXe8Z7rUJP0FgAFKkOhcHdFWzL1ZYggtwBgiB3LSoele9o3ZqFh7iCBhHbVLAnMuJ0HF8hEw7UKecE6wd-MBXgeRMdubGydhAMZSmuUjRpqplML40bmrb8VjJKNZswD1Cg",
-		"QgAAAC6zw0qH2DJtnXe8Z7rUJP0rG4sjLa_KVLlww5WEDJ__30J15en-K_6Y68jb_rU93e2TFY6fb0MYiQ1UrLNMQufqODHZUl39Lo6cXAOVOThjAMZSmuVH7n85JOYSCgzpvowMAVueGG0Xxg",
-		"203ef0f713abcebd8d62c35c0e3f12f87d71e5e4",
-		"MDEyOk9yZ2FuaXphdGlvbjU3MzI0Nzk1",
 		"730970532670-compute@developer.gserviceaccount.com",
-		"arn-aws-ecs-eu-west-2-396248696294-cluster-london-01-ECSCluster-27iuIYva8nO4", // ?
+		"819db2242a648b305395537022523d65",
+		"952bea17-3776-11ea-9341-42010a84012a",
+		"a3226860758.html",
 		"AAAA028295945",
-		"sp_ANQXRpqH_urn$3Auri$3Abase64$3A6698b0a3-97ad-52ce-8fc3-17d99e37a726",
+		"arn-aws-ecs-eu-west-2-396248696294-cluster-london-01-ECSCluster-27iuIYva8nO4",
+		"arn-aws-ecs-eu-west-2-396248696294-cluster-london-01-ECSCluster-27iuIYva8nO4", // ?
+		"bnjksfd897345nl098asd53412kl98",
+		"c738338322370b47a79251f7510dd",                        // prefixed hex
+		"ci12NC01YzkyNTEzYzllMDRhLTAtYy5tb25pdG9yaW5nLmpzb24=", // long base64
+		"css/login.0f48c49a34eb53ea4623.min.css",
+		"d_fLLxlhzDilixeBEimaZ5",
+		"e21f7112-3d3b-4632-9da3-a4af2e0e9166",
+		"e8782afc112720300c049ff124434b79",
+		"fb6cjraf9cejut2a",
+		"i-0236530c66ed02200",
+		"JEHJW4BKVFDRTMTUQLHKK5WVAU",
+		"john.dow.1981@protonmail.com",
+		"MDEyOk9yZ2FuaXphdGlvbjU3MzI0Nzk1",
+		"MNUTGVFMGLEMFTBH0XSE5E02F6J2DS",
 		"n63nd45qsj",
 		"n9z9QGNiz",
+		"NC4WTmcy",
 		"proxy.3d2100fd7107262ecb55ce6847f01fa5.html",
+		"QgAAAC6zw0qH2DJtnXe8Z7rUJP0FgAFKkOhcHdFWzL1ZYggtwBgiB3LSoele9o3ZqFh7iCBhHbVLAnMuJ0HF8hEw7UKecE6wd-MBXgeRMdubGydhAMZSmuUjRpqplML40bmrb8VjJKNZswD1Cg",
+		"QgAAAC6zw0qH2DJtnXe8Z7rUJP0rG4sjLa_KVLlww5WEDJ__30J15en-K_6Y68jb_rU93e2TFY6fb0MYiQ1UrLNMQufqODHZUl39Lo6cXAOVOThjAMZSmuVH7n85JOYSCgzpvowMAVueGG0Xxg",
+		"qwerqwerasdfqwer@protonmai.com",
 		"r-ext-5579e00a95c90",
 		"r-ext-5579e8b12f11e",
 		"r-v4-5c92513c9e04a",
 		"r-v4-5c92513c9e04a-0-c.monitoring.json",
 		"segments-1563566437171.639994",
+		"sp_ANQXRpqH_urn$3Auri$3Abase64$3A6698b0a3-97ad-52ce-8fc3-17d99e37a726",
+		"sp_dxJTfx11_576742227280287872",
+		"sp_NnUPB5wj_601a2bdcc5b69137248ddbbf",
+		"sp_NxITuoE4_premiumchron-article-14302157_c_ryGQBs_r_yIWvwP",
 		"t_52d94268-8810-4a7e-ba87-ffd657a6752f",
 		"timeouts-1563566437171.639994",
+		"u_YPF3GsGKMo02",
+
+		"0000000000 65535 f",
+		"0000000178 00000 n",
+		"0-10000",
+		"01526123,",
+		"0,18168,183955,3,4,1151616,5663,731,223,5104,207,3204,10,1051,175,364,1435,4,60,576,241,383,246,5,1102",
+		"05/10/2020",
+		"14336456724940333",
+		"fb6cjraf9cejut2a",
+		"JEHJW4BKVFDRTMTUQLHKK5WVAU",
 
 		// TODO
-		// "fb6cjraf9cejut2a",
-		// "Fxvd1timk", // questionable
-		// "JEHJW4BKVFDRTMTUQLHKK5WVAU",
+		/*
+			"0,20",
+			"0.001",
+			"YISAtiX1",
+			"Fxvd1timk", // questionable
+			"B4GCSkORAJs",
+			"D_4EDAqenHQ",
+			"EICJp29EGOk",
+			"Fxvd1timk",
+			"GTqMZELYfQQ",
+			"GZPTpLPEGmwHGWPC",
+			"_HChnE9NDPY",
+			"NwhjgIWHgGg",
+			"production/tsbqksph4xswqjexfbec",
+			"p/u/bguhrxupr23mw3nwxcrw",
+			"nRSNapbJZnc",
+			"zgfpbtolciznub5egzxk",
+			"zufnu7aimadua9wrgwwo",
+			"zznto1jzch9yjsbtbrul",
+		*/
 	}
 
 	for _, str := range cases {
@@ -69,3 +195,18 @@ func TestPositive(t *testing.T) {
 		}
 	}
 }
+
+func TestVersionStrings(t *testing.T) {
+	cases := []string{
+		"1.0",
+		"1.0.0",
+		"v2.1.3",
+		"2.1.73",
+	}
+
+	for _, str := range cases {
+		if !IsVersionString(str) {
+			t.Errorf("Mistakenly false: %s", str)
+		}
+	}
+}

agent/pkg/oas/ignores.go

@@ -12,9 +12,9 @@ var ignoredHeaders = []string{
 	"authorization", "cache-control", "connection", "content-encoding", "content-length", "content-type", "cookie",
 	"date", "dnt", "expect", "forwarded", "from", "front-end-https", "host", "http2-settings",
 	"max-forwards", "origin", "pragma", "proxy-authorization", "proxy-connection", "range", "referer",
-	"save-data", "te", "trailer", "transfer-encoding", "upgrade", "upgrade-insecure-requests",
-	"server", "user-agent", "via", "warning", "strict-transport-security",
-	"x-att-deviceid", "x-correlation-id", "correlation-id", "x-client-data",
+	"save-data", "te", "trailer", "transfer-encoding", "upgrade", "upgrade-insecure-requests", "x-download-options",
+	"server", "user-agent", "via", "warning", "strict-transport-security", "x-permitted-cross-domain-policies",
+	"x-att-deviceid", "x-correlation-id", "correlation-id", "x-client-data", "x-dns-prefetch-control",
 	"x-http-method-override", "x-real-ip", "x-request-id", "x-request-start", "x-requested-with", "x-uidh",
 	"x-same-domain", "x-content-type-options", "x-frame-options", "x-xss-protection",
 	"x-wap-profile", "x-scheme", "status", "x-cache", "x-application-context", "retry-after",
@@ -31,7 +31,7 @@ var ignoredHeaderPrefixes = []string{
 	":", "accept-", "access-control-", "if-", "sec-", "grpc-",
 	"x-forwarded-", "x-original-", "cf-",
 	"x-up9-", "x-envoy-", "x-hasura-", "x-b3-", "x-datadog-", "x-envoy-", "x-amz-", "x-newrelic-", "x-prometheus-",
-	"x-akamai-", "x-spotim-", "x-amzn-", "x-ratelimit-",
+	"x-akamai-", "x-spotim-", "x-amzn-", "x-ratelimit-", "x-goog-",
 }
 
 func isCtypeIgnored(ctype string) bool {

agent/pkg/oas/oas_generator.go

@@ -3,10 +3,11 @@ package oas
 import (
 	"context"
 	"encoding/json"
-	"github.com/up9inc/mizu/shared/logger"
-	"mizuserver/pkg/har"
 	"net/url"
 	"sync"
+
+	"github.com/up9inc/mizu/agent/pkg/har"
+	"github.com/up9inc/mizu/shared/logger"
 )
 
 var (
@@ -29,7 +30,7 @@ func (g *oasGenerator) Start() {
 	ctx, cancel := context.WithCancel(context.Background())
 	g.cancel = cancel
 	g.ctx = ctx
-	g.entriesChan = make(chan har.Entry, 100) // buffer up to 100 entries for OAS processing
+	g.entriesChan = make(chan EntryWithSource, 100) // buffer up to 100 entries for OAS processing
 	g.ServiceSpecs = &sync.Map{}
 	g.started = true
 	go instance.runGeneretor()
@@ -42,11 +43,12 @@ func (g *oasGenerator) runGeneretor() {
 			logger.Log.Infof("OAS Generator was canceled")
 			return
 
-		case entry, ok := <-g.entriesChan:
+		case entryWithSource, ok := <-g.entriesChan:
 			if !ok {
 				logger.Log.Infof("OAS Generator - entries channel closed")
 				break
 			}
+			entry := entryWithSource.Entry
 			u, err := url.Parse(entry.Request.URL)
 			if err != nil {
 				logger.Log.Errorf("Failed to parse entry URL: %v, err: %v", entry.Request.URL, err)
@@ -61,7 +63,7 @@ func (g *oasGenerator) runGeneretor() {
 				gen = val.(*SpecGen)
 			}
 
-			opId, err := gen.feedEntry(entry)
+			opId, err := gen.feedEntry(entryWithSource)
 			if err != nil {
 				txt, suberr := json.Marshal(entry)
 				if suberr == nil {
@@ -77,12 +79,12 @@ func (g *oasGenerator) runGeneretor() {
 	}
 }
 
-func (g *oasGenerator) PushEntry(entry *har.Entry) {
+func (g *oasGenerator) PushEntry(entryWithSource *EntryWithSource) {
 	if !g.started {
 		return
 	}
 	select {
-	case g.entriesChan <- *entry:
+	case g.entriesChan <- *entryWithSource:
 	default:
 		logger.Log.Warningf("OAS Generator - entry wasn't sent to channel because the channel has no buffer or there is no receiver")
 	}
@@ -98,10 +100,15 @@ func newOasGenerator() *oasGenerator {
 	}
 }
 
+type EntryWithSource struct {
+	Source string
+	Entry  har.Entry
+}
+
 type oasGenerator struct {
 	started      bool
 	ctx          context.Context
 	cancel       context.CancelFunc
 	ServiceSpecs *sync.Map
-	entriesChan  chan har.Entry
+	entriesChan  chan EntryWithSource
 }

agent/pkg/oas/specgen.go

@@ -3,18 +3,26 @@ package oas
 import (
 	"encoding/json"
 	"errors"
+	"fmt"
+	"github.com/chanced/openapi"
+	"github.com/google/uuid"
+	"github.com/nav-inc/datetime"
+	"github.com/up9inc/mizu/shared/logger"
 	"mime"
-	"mizuserver/pkg/har"
 	"net/url"
 	"strconv"
 	"strings"
 	"sync"
 
-	"github.com/chanced/openapi"
-	"github.com/google/uuid"
-	"github.com/up9inc/mizu/shared/logger"
+	"github.com/up9inc/mizu/agent/pkg/har"
+
+	"time"
 )
 
+const LastSeenTS = "x-last-seen-ts"
+const CountersTotal = "x-counters-total"
+const CountersPerSource = "x-counters-per-source"
+
 type reqResp struct { // hello, generics in Go
 	Req  *har.Request
 	Resp *har.Response
@@ -44,17 +52,23 @@ func NewGen(server string) *SpecGen {
 
 func (g *SpecGen) StartFromSpec(oas *openapi.OpenAPI) {
 	g.oas = oas
+	g.tree = new(Node)
 	for pathStr, pathObj := range oas.Paths.Items {
 		pathSplit := strings.Split(string(pathStr), "/")
 		g.tree.getOrSet(pathSplit, pathObj)
+
+		// clean "last entry timestamp" markers from the past
+		for _, pathAndOp := range g.tree.listOps() {
+			delete(pathAndOp.op.Extensions, LastSeenTS)
+		}
 	}
 }
 
-func (g *SpecGen) feedEntry(entry har.Entry) (string, error) {
+func (g *SpecGen) feedEntry(entryWithSource EntryWithSource) (string, error) {
 	g.lock.Lock()
 	defer g.lock.Unlock()
 
-	opId, err := g.handlePathObj(&entry)
+	opId, err := g.handlePathObj(&entryWithSource)
 	if err != nil {
 		return "", err
 	}
@@ -69,10 +83,23 @@ func (g *SpecGen) GetSpec() (*openapi.OpenAPI, error) {
 
 	g.tree.compact()
 
-	for _, pathop := range g.tree.listOps() {
-		if pathop.op.Summary == "" {
-			pathop.op.Summary = pathop.path
+	counters := CounterMaps{counterTotal: Counter{}, counterMapTotal: CounterMap{}}
+
+	for _, pathAndOp := range g.tree.listOps() {
+		opObj := pathAndOp.op
+		if opObj.Summary == "" {
+			opObj.Summary = pathAndOp.path
 		}
+
+		err := counters.processOp(opObj)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	err := counters.processOas(g.oas)
+	if err != nil {
+		return nil, err
 	}
 
 	// put paths back from tree into OAS
@@ -80,6 +107,8 @@ func (g *SpecGen) GetSpec() (*openapi.OpenAPI, error) {
 
 	suggestTags(g.oas)
 
+	g.oas.Info.Description = setCounterMsgIfOk(g.oas.Info.Description, &counters.counterTotal)
+
 	// to make a deep copy, no better idea than marshal+unmarshal
 	specText, err := json.MarshalIndent(g.oas, "", "\t")
 	if err != nil {
@@ -131,22 +160,6 @@ func suggestTags(oas *openapi.OpenAPI) {
 	}
 }
 
-func getSimilarPrefix(strs []string) string {
-	chunked := make([][]string, 0)
-	for _, item := range strs {
-		chunked = append(chunked, strings.Split(item, "/"))
-	}
-
-	cmn := longestCommonXfix(chunked, true)
-	res := make([]string, 0)
-	for _, chunk := range cmn {
-		if chunk != "api" && !IsVersionString(chunk) && !strings.HasPrefix(chunk, "{") {
-			res = append(res, chunk)
-		}
-	}
-	return strings.Join(res[1:], ".")
-}
-
 func deleteFromSlice(s []string, val string) []string {
 	temp := s[:0]
 	for _, x := range s {
@@ -168,7 +181,8 @@ func getPathsKeys(mymap map[openapi.PathValue]*openapi.PathObj) []string {
 	return keys
 }
 
-func (g *SpecGen) handlePathObj(entry *har.Entry) (string, error) {
+func (g *SpecGen) handlePathObj(entryWithSource *EntryWithSource) (string, error) {
+	entry := entryWithSource.Entry
 	urlParsed, err := url.Parse(entry.Request.URL)
 	if err != nil {
 		return "", err
@@ -212,7 +226,7 @@ func (g *SpecGen) handlePathObj(entry *har.Entry) (string, error) {
 		split = strings.Split(urlParsed.Path, "/")
 	}
 	node := g.tree.getOrSet(split, new(openapi.PathObj))
-	opObj, err := handleOpObj(entry, node.pathObj)
+	opObj, err := handleOpObj(entryWithSource, node.pathObj)
 
 	if opObj != nil {
 		return opObj.OperationID, err
@@ -221,7 +235,8 @@ func (g *SpecGen) handlePathObj(entry *har.Entry) (string, error) {
 	return "", err
 }
 
-func handleOpObj(entry *har.Entry, pathObj *openapi.PathObj) (*openapi.Operation, error) {
+func handleOpObj(entryWithSource *EntryWithSource, pathObj *openapi.PathObj) (*openapi.Operation, error) {
+	entry := entryWithSource.Entry
 	isSuccess := 100 <= entry.Response.Status && entry.Response.Status < 400
 	opObj, wasMissing, err := getOpObj(pathObj, entry.Request.Method, isSuccess)
 	if err != nil {
@@ -243,9 +258,86 @@ func handleOpObj(entry *har.Entry, pathObj *openapi.PathObj) (*openapi.Operation
 		return nil, err
 	}
 
+	err = handleCounters(opObj, isSuccess, entryWithSource)
+	if err != nil {
+		return nil, err
+	}
+
 	return opObj, nil
 }
 
+func handleCounters(opObj *openapi.Operation, success bool, entryWithSource *EntryWithSource) error {
+	// TODO: if performance around DecodeExtension+SetExtension is bad, store counters as separate maps
+	counter := Counter{}
+	counterMap := CounterMap{}
+	prevTs := 0.0
+	if opObj.Extensions == nil {
+		opObj.Extensions = openapi.Extensions{}
+	} else {
+		if _, ok := opObj.Extensions.Extension(CountersTotal); ok {
+			err := opObj.Extensions.DecodeExtension(CountersTotal, &counter)
+			if err != nil {
+				return err
+			}
+		}
+
+		if _, ok := opObj.Extensions.Extension(CountersPerSource); ok {
+			err := opObj.Extensions.DecodeExtension(CountersPerSource, &counterMap)
+			if err != nil {
+				return err
+			}
+		}
+
+		if _, ok := opObj.Extensions.Extension(LastSeenTS); ok {
+			err := opObj.Extensions.DecodeExtension(LastSeenTS, &prevTs)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	var counterPerSource *Counter
+	if existing, ok := counterMap[entryWithSource.Source]; ok {
+		counterPerSource = existing
+	} else {
+		counterPerSource = new(Counter)
+		counterMap[entryWithSource.Source] = counterPerSource
+	}
+
+	started, err := datetime.Parse(entryWithSource.Entry.StartedDateTime, time.UTC)
+	if err != nil {
+		return err
+	}
+
+	ts := float64(started.UnixNano()) / float64(time.Millisecond) / 1000
+	rt := float64(entryWithSource.Entry.Time) / 1000
+
+	dur := 0.0
+	if prevTs != 0 {
+		dur = ts - prevTs
+	}
+
+	counter.addEntry(ts, rt, success, dur)
+	counterPerSource.addEntry(ts, rt, success, dur)
+
+	err = opObj.Extensions.SetExtension(LastSeenTS, ts)
+	if err != nil {
+		return err
+	}
+
+	err = opObj.Extensions.SetExtension(CountersTotal, counter)
+	if err != nil {
+		return err
+	}
+
+	err = opObj.Extensions.SetExtension(CountersPerSource, counterMap)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
 func handleRequest(req *har.Request, opObj *openapi.Operation, isSuccess bool) error {
 	// TODO: we don't handle the situation when header/qstr param can be defined on pathObj level. Also the path param defined on opObj
 
@@ -499,3 +591,56 @@ func getOpObj(pathObj *openapi.PathObj, method string, createIfNone bool) (*open
 
 	return *op, isMissing, nil
 }
+
+type CounterMaps struct {
+	counterTotal    Counter
+	counterMapTotal CounterMap
+}
+
+func (m *CounterMaps) processOp(opObj *openapi.Operation) error {
+	if _, ok := opObj.Extensions.Extension(CountersTotal); ok {
+		counter := new(Counter)
+		err := opObj.Extensions.DecodeExtension(CountersTotal, counter)
+		if err != nil {
+			return err
+		}
+		m.counterTotal.addOther(counter)
+
+		opObj.Description = setCounterMsgIfOk(opObj.Description, counter)
+	}
+
+	if _, ok := opObj.Extensions.Extension(CountersPerSource); ok {
+		counterMap := new(CounterMap)
+		err := opObj.Extensions.DecodeExtension(CountersPerSource, counterMap)
+		if err != nil {
+			return err
+		}
+		m.counterMapTotal.addOther(counterMap)
+	}
+	return nil
+}
+
+func (m *CounterMaps) processOas(oas *openapi.OpenAPI) error {
+	if oas.Extensions == nil {
+		oas.Extensions = openapi.Extensions{}
+	}
+
+	err := oas.Extensions.SetExtension(CountersTotal, m.counterTotal)
+	if err != nil {
+		return err
+	}
+
+	err = oas.Extensions.SetExtension(CountersPerSource, m.counterMapTotal)
+	if err != nil {
+		return nil
+	}
+	return nil
+}
+
+func setCounterMsgIfOk(oldStr string, cnt *Counter) string {
+	tpl := "Mizu observed %d entries (%d failed), at %.3f hits/s, average response time is %.3f seconds"
+	if oldStr == "" || (strings.HasPrefix(oldStr, "Mizu ") && strings.HasSuffix(oldStr, " seconds")) {
+		return fmt.Sprintf(tpl, cnt.Entries, cnt.Failures, cnt.SumDuration/float64(cnt.Entries), cnt.SumRT/float64(cnt.Entries))
+	}
+	return oldStr
+}

agent/pkg/oas/specgen_test.go

@@ -2,15 +2,16 @@ package oas
 
 import (
 	"encoding/json"
-	"github.com/chanced/openapi"
-	"github.com/op/go-logging"
-	"github.com/up9inc/mizu/shared/logger"
 	"io/ioutil"
-	"mizuserver/pkg/har"
 	"os"
 	"strings"
 	"testing"
 	"time"
+
+	"github.com/chanced/openapi"
+	"github.com/op/go-logging"
+	"github.com/up9inc/mizu/agent/pkg/har"
+	"github.com/up9inc/mizu/shared/logger"
 )
 
 // if started via env, write file into subdir

agent/pkg/oas/test_artifacts/example.ldjson

@@ -1,4 +1,4 @@
-{"messageType": "http", "_source": null, "firstMessageTime": 1627298057.784151, "lastMessageTime": 1627298065.729303, "messageCount": 12}
+{"messageType": "http", "_source": "some-source", ",firstMessageTime": 1627298057.784151, "lastMessageTime": 1627298065.729303, "messageCount": 12}
 {"_id": "", "startedDateTime": "2021-07-26T11:14:17.78415179Z", "time": 13, "request": {"method": "GET", "url": "http://catalogue/catalogue/size?tags=", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "x-some", "value": "demo val"},{"name": "Host", "value": "catalogue"}, {"name": "Connection", "value": "close"}], "queryString": [{"name": "tags", "value": ""}], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Content-Type", "value": "application/json; charset=utf-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:17 GMT"}, {"name": "Content-Length", "value": "22"}], "content": {"size": 22, "mimeType": "application/json; charset=utf-8", "text": "eyJlcnIiOm51bGwsInNpemUiOjl9", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": 22}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 13}}
 {"_id": "", "startedDateTime": "2021-07-26T11:14:17.784918698Z", "time": 19, "request": {"method": "GET", "url": "http://catalogue/catalogue?page=1&size=6&tags=", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Connection", "value": "close"}, {"name": "Host", "value": "catalogue"}], "queryString": [{"name": "page", "value": "1"}, {"name": "size", "value": "6"}, {"name": "tags", "value": ""}], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Content-Type", "value": "application/json; charset=utf-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:17 GMT"}, {"name": "Content-Length", "value": "1927"}], "content": {"size": 1927, "mimeType": "application/json; charset=utf-8", "text": "W3siaWQiOiIwM2ZlZjZhYy0xODk2LTRjZTgtYmQ2OS1iNzk4Zjg1YzZlMGIiLCJuYW1lIjoiSG9seSIsImRlc2NyaXB0aW9uIjoiU29ja3MgZml0IGZvciBhIE1lc3NpYWguIFlvdSB0b28gY2FuIGV4cGVyaWVuY2Ugd2Fsa2luZyBpbiB3YXRlciB3aXRoIHRoZXNlIHNwZWNpYWwgZWRpdGlvbiBiZWF1dGllcy4gRWFjaCBob2xlIGlzIGxvdmluZ2x5IHByb2dnbGVkIHRvIGxlYXZlIHNtb290aCBlZGdlcy4gVGhlIG9ubHkgc29jayBhcHByb3ZlZCBieSBhIGhpZ2hlciBwb3dlci4iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9ob2x5XzEuanBlZyIsIi9jYXRhbG9ndWUvaW1hZ2VzL2hvbHlfMi5qcGVnIl0sInByaWNlIjo5OS45OSwiY291bnQiOjEsInRhZyI6WyJhY3Rpb24iLCJtYWdpYyJdfSx7ImlkIjoiMzM5NWE0M2UtMmQ4OC00MGRlLWI5NWYtZTAwZTE1MDIwODViIiwibmFtZSI6IkNvbG91cmZ1bCIsImRlc2NyaXB0aW9uIjoicHJvaWRlbnQgb2NjYWVjYXQgaXJ1cmUgZXQgZXhjZXB0ZXVyIGxhYm9yZSBtaW5pbSBuaXNpIGFtZXQgaXJ1cmUiLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9jb2xvdXJmdWxfc29ja3MuanBnIiwiL2NhdGFsb2d1ZS9pbWFnZXMvY29sb3VyZnVsX3NvY2tzLmpwZyJdLCJwcmljZSI6MTgsImNvdW50Ijo0MzgsInRhZyI6WyJicm93biIsImJsdWUiXX0seyJpZCI6IjUxMGEwZDdlLThlODMtNDE5My1iNDgzLWUyN2UwOWRkYzM0ZCIsIm5hbWUiOiJTdXBlclNwb3J0IFhMIiwiZGVzY3JpcHRpb24iOiJSZWFkeSBmb3IgYWN0aW9uLiBFbmdpbmVlcnM6IGJlIHJlYWR5IHRvIHNtYXNoIHRoYXQgbmV4dCBidWchIEJlIHJlYWR5LCB3aXRoIHRoZXNlIHN1cGVyLWFjdGlvbi1zcG9ydC1tYXN0ZXJwaWVjZXMuIFRoaXMgcGFydGljdWxhciBlbmdpbmVlciB3YXMgY2hhc2VkIGF3YXkgZnJvbSB0aGUgb2ZmaWNlIHdpdGggYSBzdGljay4iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9wdW1hXzEuanBlZyIsIi9jYXRhbG9ndWUvaW1hZ2VzL3B1bWFfMi5qcGVnIl0sInByaWNlIjoxNSwiY291bnQiOjgyMCwidGFnIjpbInNwb3J0IiwiZm9ybWFsIiwiYmxhY2siXX0seyJpZCI6IjgwOGEyZGUxLTFhYWEtNGMyNS1hOWI5LTY2MTJlOGYyOWEzOCIsIm5hbWUiOiJDcm9zc2VkIiwiZGVzY3JpcHRpb24iOiJBIG1hdHVyZSBzb2NrLCBjcm9zc2VkLCB3aXRoIGFuIGFpciBvZiBub25jaGFsYW5jZS4iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9jcm9zc18xLmpwZWciLCIvY2F0YWxvZ3VlL2ltYWdlcy9jcm9zc18yLmpwZWciXSwicHJpY2UiOjE3LjMyLCJjb3VudCI6NzM4LCJ0YWciOlsiYmx1ZSIsImFjdGlvbiIsInJlZCIsImZvcm1hbCJdfSx7ImlkIjoiODE5ZTFmYmYtOGI3ZS00ZjZkLTgxMWYtNjkzNTM0OTE2YThiIiwibmFtZSI6IkZpZ3Vlcm9hIiwiZGVzY3JpcHRpb24iOiJlbmltIG9mZmljaWEgYWxpcXVhIGV4Y2VwdGV1ciBlc3NlIGRlc2VydW50IHF1aXMgYWxpcXVpcCBub3N0cnVkIGFuaW0iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9XQVQuanBnIiwiL2NhdGFsb2d1ZS9pbWFnZXMvV0FUMi5qcGciXSwicHJpY2UiOjE0LCJjb3VudCI6ODA4LCJ0YWciOlsiZ3JlZW4iLCJmb3JtYWwiLCJibHVlIl19LHsiaWQiOiI4MzdhYjE0MS0zOTllLTRjMWYtOWFiYy1iYWNlNDAyOTZiYWMiLCJuYW1lIjoiQ2F0IHNvY2tzIiwiZGVzY3JpcHRpb24iOiJjb25zZXF1YXQgYW1ldCBjdXBpZGF0YXQgbWluaW0gbGFib3J1bSB0ZW1wb3IgZWxpdCBleCBjb25zZXF1YXQgaW4iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9jYXRzb2Nrcy5qcGciLCIvY2F0YWxvZ3VlL2ltYWdlcy9jYXRzb2NrczIuanBnIl0sInByaWNlIjoxNSwiY291bnQiOjE3NSwidGFnIjpbImJyb3duIiwiZm9ybWFsIiwiZ3JlZW4iXX1dCg==", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": 1927}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 19}}
 {"_id": "", "startedDateTime": "2021-07-26T11:14:17.78418182Z", "time": 7, "request": {"method": "GET", "url": "http://catalogue/tags", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Connection", "value": "close"}, {"name": "Host", "value": "catalogue"}], "queryString": [], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Content-Type", "value": "application/json; charset=utf-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:17 GMT"}, {"name": "Content-Length", "value": "107"}], "content": {"size": 107, "mimeType": "application/json; charset=utf-8", "text": "eyJlcnIiOm51bGwsInRhZ3MiOlsiYnJvd24iLCJnZWVrIiwiZm9ybWFsIiwiYmx1ZSIsInNraW4iLCJyZWQiLCJhY3Rpb24iLCJzcG9ydCIsImJsYWNrIiwibWFnaWMiLCJncmVlbiJdfQ==", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": 107}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 7}}

agent/pkg/oas/test_artifacts/params.har

@@ -121,6 +121,120 @@
           "connect": 262,
           "ssl": -1
         }
+      },
+      {
+        "startedDateTime": "2019-09-06T06:16:20.047122+00:00",
+        "time": 630,
+        "request": {
+          "method": "GET",
+          "url": "https://httpbin.org/appears-once",
+          "httpVersion": "HTTP/1.1",
+          "cookies": [],
+          "headers": [],
+          "queryString": [],
+          "headersSize": 1542,
+          "bodySize": 0
+        },
+        "response": {
+          "status": 200,
+          "statusText": "OK",
+          "httpVersion": "HTTP/1.1",
+          "cookies": [],
+          "headers": [],
+          "content": {
+            "size": 39,
+            "compression": -20,
+            "mimeType": "application/json",
+            "text": "null"
+          },
+          "redirectURL": "",
+          "headersSize": 248,
+          "bodySize": 39
+        },
+        "cache": {},
+        "timings": {
+          "send": 14,
+          "receive": 4,
+          "wait": 350,
+          "connect": 262,
+          "ssl": -1
+        }
+      },
+      {
+        "startedDateTime": "2019-09-06T06:16:20.747122+00:00",
+        "time": 630,
+        "request": {
+          "method": "GET",
+          "url": "https://httpbin.org/appears-twice",
+          "httpVersion": "HTTP/1.1",
+          "cookies": [],
+          "headers": [],
+          "queryString": [],
+          "headersSize": 1542,
+          "bodySize": 0
+        },
+        "response": {
+          "status": 200,
+          "statusText": "OK",
+          "httpVersion": "HTTP/1.1",
+          "cookies": [],
+          "headers": [],
+          "content": {
+            "size": 39,
+            "compression": -20,
+            "mimeType": "application/json",
+            "text": "null"
+          },
+          "redirectURL": "",
+          "headersSize": 248,
+          "bodySize": 39
+        },
+        "cache": {},
+        "timings": {
+          "send": 14,
+          "receive": 4,
+          "wait": 350,
+          "connect": 262,
+          "ssl": -1
+        }
+      },
+      {
+        "startedDateTime": "2019-09-06T06:16:21.747122+00:00",
+        "time": 630,
+        "request": {
+          "method": "GET",
+          "url": "https://httpbin.org/appears-twice",
+          "httpVersion": "HTTP/1.1",
+          "cookies": [],
+          "headers": [],
+          "queryString": [],
+          "headersSize": 1542,
+          "bodySize": 0
+        },
+        "response": {
+          "status": 200,
+          "statusText": "OK",
+          "httpVersion": "HTTP/1.1",
+          "cookies": [],
+          "headers": [],
+          "content": {
+            "size": 39,
+            "compression": -20,
+            "mimeType": "application/json",
+            "text": "null"
+          },
+          "redirectURL": "",
+          "headersSize": 248,
+          "bodySize": 39
+        },
+        "cache": {},
+        "timings": {
+          "send": 14,
+          "receive": 4,
+          "wait": 350,
+          "connect": 262,
+          "ssl": -1
+        }
       }
     ]
   }

agent/pkg/oas/tree.go

@@ -62,7 +62,6 @@ func (n *Node) getOrSet(path NodePath, existingPathObj *openapi.PathObj) (node *
 		if paramObj != nil {
 			node.pathParam = paramObj
 		} else if chunkIsGibberish {
-
 			newParam := n.createParam()
 			node.pathParam = newParam
 		} else {
@@ -100,11 +99,14 @@ func (n *Node) createParam() *openapi.ParameterObj {
 		} else if strings.HasSuffix(*n.constant, "s") && len(*n.constant) > 3 {
 			name = *n.constant
 			name = name[:len(name)-1] + "Id"
-		} else if isAlpha(*n.constant) {
+		} else {
 			name = *n.constant + "Id"
 		}
 
-		name = cleanNonAlnum([]byte(name))
+		name = cleanStr(name, isAlNumRune)
+		if !isAlphaRune(rune(name[0])) {
+			name = "_" + name
+		}
 	}
 
 	newParam := createSimpleParam(name, "path", "string")

agent/pkg/oas/tree_test.go

@@ -15,7 +15,7 @@ func TestTree(t *testing.T) {
 		{"/", 0, ""},
 		{"/v1.0.0/config/launcher/sp_nKNHCzsN/f34efcae-6583-11eb-908a-00b0fcb9d4f6/vendor,init,conversation", 1, "vendor,init,conversation"},
 		{"/v1.0.0/config/launcher/sp_nKNHCzsN/{f34efcae-6583-11eb-908a-00b0fcb9d4f6}/vendor,init,conversation", 0, "vendor,init,conversation"},
-		{"/getSvgs/size/small/brand/SFLY/layoutId/170943/layoutVersion/1/sizeId/742/surface/0/isLandscape/true/childSkus/%7B%7D", 1, ""},
+		{"/getSvgs/size/small/brand/SFLY/layoutId/170943/layoutVersion/1/sizeId/742/surface/0/isLandscape/true/childSkus/%7B%7D", 1, "{}"},
 	}
 
 	tree := new(Node)
@@ -23,7 +23,7 @@ func TestTree(t *testing.T) {
 		split := strings.Split(tc.inp, "/")
 		pathObj := new(openapi.PathObj)
 		node := tree.getOrSet(split, pathObj)
-		
+
 		fillPathParams(node, pathObj)
 
 		if node.constant != nil && *node.constant != tc.label {

agent/pkg/oas/utils.go

@@ -3,10 +3,11 @@ package oas
 import (
 	"encoding/json"
 	"errors"
-	"mizuserver/pkg/har"
 	"strconv"
 	"strings"
 
+	"github.com/up9inc/mizu/agent/pkg/har"
+
 	"github.com/chanced/openapi"
 	"github.com/up9inc/mizu/shared/logger"
 )
@@ -260,6 +261,22 @@ func longestCommonXfix(strs [][]string, pre bool) []string { // https://github.c
 	return xfix
 }
 
+func getSimilarPrefix(strs []string) string {
+	chunked := make([][]string, 0)
+	for _, item := range strs {
+		chunked = append(chunked, strings.Split(item, "/"))
+	}
+
+	cmn := longestCommonXfix(chunked, true)
+	res := make([]string, 0)
+	for _, chunk := range cmn {
+		if chunk != "api" && !IsVersionString(chunk) && !strings.HasPrefix(chunk, "{") {
+			res = append(res, chunk)
+		}
+	}
+	return strings.Join(res[1:], ".")
+}
+
 // returns all non-nil ops in PathObj
 func getOps(pathObj *openapi.PathObj) []*openapi.Operation {
 	ops := []**openapi.Operation{&pathObj.Get, &pathObj.Patch, &pathObj.Put, &pathObj.Options, &pathObj.Post, &pathObj.Trace, &pathObj.Head, &pathObj.Delete}
@@ -323,13 +340,11 @@ func anyJSON(text string) (anyVal interface{}, isJSON bool) {
 	return nil, false
 }
 
-func cleanNonAlnum(s []byte) string {
+func cleanStr(str string, criterion func(r rune) bool) string {
+	s := []byte(str)
 	j := 0
 	for _, b := range s {
-		if ('a' <= b && b <= 'z') ||
-			('A' <= b && b <= 'Z') ||
-			('0' <= b && b <= '9') ||
-			b == ' ' {
+		if criterion(rune(b)) {
 			s[j] = b
 			j++
 		}
@@ -337,11 +352,21 @@ func cleanNonAlnum(s []byte) string {
 	return string(s[:j])
 }
 
+/*
 func isAlpha(s string) bool {
 	for _, r := range s {
-		if (r < 'a' || r > 'z') && (r < 'A' || r > 'Z') {
+		if isAlphaRune(r) {
 			return false
 		}
 	}
 	return true
 }
+*/
+
+func isAlphaRune(r rune) bool {
+	return !((r < 'a' || r > 'z') && (r < 'A' || r > 'Z'))
+}
+
+func isAlNumRune(b rune) bool {
+	return isAlphaRune(b) || ('0' <= b && b <= '9')
+}

agent/pkg/oas/utils_test.go

@@ -33,3 +33,27 @@ func TestAnyJSON(t *testing.T) {
 		}
 	}
 }
+
+func TestStrRunes(t *testing.T) {
+	if isAlphaRune('5') {
+		t.Logf("Failed")
+	}
+	if !isAlphaRune('a') {
+		t.Logf("Failed")
+	}
+
+	if !isAlNumRune('5') {
+		t.Logf("Failed")
+	}
+	if isAlNumRune(' ') {
+		t.Logf("Failed")
+	}
+
+	if cleanStr("-abc_567", isAlphaRune) != "abc" {
+		t.Logf("Failed")
+	}
+
+	if cleanStr("-abc_567", isAlNumRune) != "abc567" {
+		t.Logf("Failed")
+	}
+}

agent/pkg/providers/install_provider.go

@@ -3,7 +3,8 @@ package providers
 import (
 	"context"
 	"errors"
-	"mizuserver/pkg/config"
+
+	"github.com/up9inc/mizu/agent/pkg/config"
 
 	"github.com/up9inc/mizu/shared/logger"
 

agent/pkg/providers/stats_provider_test.go

@@ -2,8 +2,9 @@ package providers_test
 
 import (
 	"fmt"
-	"mizuserver/pkg/providers"
 	"testing"
+
+	"github.com/up9inc/mizu/agent/pkg/providers"
 )
 
 func TestNoEntryAddedCount(t *testing.T) {

agent/pkg/providers/status_provider.go

@@ -3,12 +3,13 @@ package providers
 import (
 	"encoding/json"
 	"fmt"
-	"github.com/patrickmn/go-cache"
-	"github.com/up9inc/mizu/shared"
-	"github.com/up9inc/mizu/tap"
-	"mizuserver/pkg/models"
 	"os"
 	"time"
+
+	"github.com/patrickmn/go-cache"
+	"github.com/up9inc/mizu/agent/pkg/models"
+	"github.com/up9inc/mizu/shared"
+	"github.com/up9inc/mizu/tap"
 )
 
 const tlsLinkRetainmentTime = time.Minute * 15

agent/pkg/providers/tapConfig/tap_config_provider.go

@@ -1,12 +1,13 @@
 package tapConfig
 
 import (
-	"github.com/up9inc/mizu/shared"
-	"github.com/up9inc/mizu/shared/logger"
-	"mizuserver/pkg/models"
-	"mizuserver/pkg/utils"
 	"os"
 	"sync"
+
+	"github.com/up9inc/mizu/agent/pkg/models"
+	"github.com/up9inc/mizu/agent/pkg/utils"
+	"github.com/up9inc/mizu/shared"
+	"github.com/up9inc/mizu/shared/logger"
 )
 
 const FilePath = shared.DataDirPath + "tap-config.json"

agent/pkg/providers/tappedPods/tapped_pods_provider.go

@@ -1,13 +1,14 @@
 package tappedPods
 
 import (
-	"github.com/up9inc/mizu/shared"
-	"github.com/up9inc/mizu/shared/logger"
-	"mizuserver/pkg/providers/tappers"
-	"mizuserver/pkg/utils"
 	"os"
 	"strings"
 	"sync"
+
+	"github.com/up9inc/mizu/agent/pkg/providers/tappers"
+	"github.com/up9inc/mizu/agent/pkg/utils"
+	"github.com/up9inc/mizu/shared"
+	"github.com/up9inc/mizu/shared/logger"
 )
 
 const FilePath = shared.DataDirPath + "tapped-pods.json"

agent/pkg/providers/tappers/tappers_provider.go

@@ -1,11 +1,12 @@
 package tappers
 
 import (
-	"github.com/up9inc/mizu/shared"
-	"github.com/up9inc/mizu/shared/logger"
-	"mizuserver/pkg/utils"
 	"os"
 	"sync"
+
+	"github.com/up9inc/mizu/agent/pkg/utils"
+	"github.com/up9inc/mizu/shared"
+	"github.com/up9inc/mizu/shared/logger"
 )
 
 const FilePath = shared.DataDirPath + "tappers-status.json"

agent/pkg/providers/user_role_provider.go

@@ -15,7 +15,8 @@ more on keto here: https://www.ory.sh/keto/docs/
 
 import (
 	"fmt"
-	"mizuserver/pkg/utils"
+
+	"github.com/up9inc/mizu/agent/pkg/utils"
 
 	ketoClient "github.com/ory/keto-client-go/client"
 	ketoRead "github.com/ory/keto-client-go/client/read"

agent/pkg/routes/config_routes.go

@@ -1,8 +1,8 @@
 package routes
 
 import (
-	"mizuserver/pkg/controllers"
-	"mizuserver/pkg/middlewares"
+	"github.com/up9inc/mizu/agent/pkg/controllers"
+	"github.com/up9inc/mizu/agent/pkg/middlewares"
 
 	"github.com/gin-gonic/gin"
 )

agent/pkg/routes/entries_routes.go

@@ -1,8 +1,8 @@
 package routes
 
 import (
-	"mizuserver/pkg/controllers"
-	"mizuserver/pkg/middlewares"
+	"github.com/up9inc/mizu/agent/pkg/controllers"
+	"github.com/up9inc/mizu/agent/pkg/middlewares"
 
 	"github.com/gin-gonic/gin"
 )

agent/pkg/routes/install_routes.go

@@ -1,7 +1,7 @@
 package routes
 
 import (
-	"mizuserver/pkg/controllers"
+	"github.com/up9inc/mizu/agent/pkg/controllers"
 
 	"github.com/gin-gonic/gin"
 )

agent/pkg/routes/metadata_routes.go

@@ -1,7 +1,7 @@
 package routes
 
 import (
-	"mizuserver/pkg/controllers"
+	"github.com/up9inc/mizu/agent/pkg/controllers"
 
 	"github.com/gin-gonic/gin"
 )

agent/pkg/routes/oas_routes.go

@@ -1,8 +1,8 @@
 package routes
 
 import (
-	"mizuserver/pkg/controllers"
-	"mizuserver/pkg/middlewares"
+	"github.com/up9inc/mizu/agent/pkg/controllers"
+	"github.com/up9inc/mizu/agent/pkg/middlewares"
 
 	"github.com/gin-gonic/gin"
 )

agent/pkg/routes/query_routes.go

@@ -1,8 +1,8 @@
 package routes
 
 import (
-	"mizuserver/pkg/controllers"
-	"mizuserver/pkg/middlewares"
+	"github.com/up9inc/mizu/agent/pkg/controllers"
+	"github.com/up9inc/mizu/agent/pkg/middlewares"
 
 	"github.com/gin-gonic/gin"
 )

agent/pkg/routes/service_map_routes.go

@@ -1,8 +1,8 @@
 package routes
 
 import (
-	"mizuserver/pkg/controllers"
-	"mizuserver/pkg/middlewares"
+	"github.com/up9inc/mizu/agent/pkg/controllers"
+	"github.com/up9inc/mizu/agent/pkg/middlewares"
 
 	"github.com/gin-gonic/gin"
 )

agent/pkg/routes/status_routes.go

@@ -1,8 +1,8 @@
 package routes
 
 import (
-	"mizuserver/pkg/controllers"
-	"mizuserver/pkg/middlewares"
+	"github.com/up9inc/mizu/agent/pkg/controllers"
+	"github.com/up9inc/mizu/agent/pkg/middlewares"
 
 	"github.com/gin-gonic/gin"
 )

agent/pkg/routes/user_routes.go

@@ -1,7 +1,7 @@
 package routes
 
 import (
-	"mizuserver/pkg/controllers"
+	"github.com/up9inc/mizu/agent/pkg/controllers"
 
 	"github.com/gin-gonic/gin"
 )

agent/pkg/rules/rulesHTTP.go

@@ -4,11 +4,12 @@ import (
 	"encoding/base64"
 	"encoding/json"
 	"fmt"
-	"mizuserver/pkg/har"
 	"reflect"
 	"regexp"
 	"strings"
 
+	"github.com/up9inc/mizu/agent/pkg/har"
+
 	"github.com/up9inc/mizu/shared/logger"
 
 	"github.com/up9inc/mizu/shared"

agent/pkg/up9/main.go

@@ -6,8 +6,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
-	"mizuserver/pkg/har"
-	"mizuserver/pkg/utils"
 	"net/http"
 	"net/url"
 	"regexp"
@@ -15,6 +13,9 @@ import (
 	"sync"
 	"time"
 
+	"github.com/up9inc/mizu/agent/pkg/har"
+	"github.com/up9inc/mizu/agent/pkg/utils"
+
 	basenine "github.com/up9inc/basenine/client/go"
 	"github.com/up9inc/mizu/shared"
 	"github.com/up9inc/mizu/shared/logger"

docs/PERMISSIONS.md

@@ -4,7 +4,7 @@
 
 This document describes in details all permissions required for full and correct operation of Mizu.
 
-## Editting permissions
+## Editing permissions
 
 During installation, Mizu creates a `ServiceAccount` and the roles it requires. No further action is required.
 However, if there is a need, it is possible to make changes to Mizu permissions.

ui/src/components/Filters.tsx

@@ -59,11 +59,9 @@ export const QueryForm: React.FC<QueryFormProps> = ({backgroundColor, ws, openWe
     const handleCloseModal = () => setOpenModal(false);
 
     const handleChange = async (e) => {
-        setQuery(e.target.value);
+        setQuery(e.target.value.trim());
     }
 
-    
-
     const handleSubmit = (e) => {
         ws.close();
         if (query) {