github/kubeshark
1
0
Fork 0
mirror of https://github.com/kubeshark/kubeshark.git synced 2025-08-12 05:43:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

🚚 Move agent directory to kubeshark/hub and use kubeshark/hub Docker image instead (#1249)

Browse Source 
* Remove the `agent` directory

* Use the new `kubeshark/hub` Docker image

* Remove `Dockerfile`

* Update `Makefile`

* Fix linter

* Change `api-server` suffix to `hub`
This commit is contained in:
M. Mert Yildiran 2022-11-24 17:18:35 -08:00 committed by GitHub
parent 8868a4c979
commit f87aa467b8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
88 changed files with 6 additions and 47477 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
.github/workflows/linter.yml vendored
View File

@ -24,13 +24,6 @@ jobs:
with:
go-version: '^1.17'
- name: Go lint - agent
uses: golangci/golangci-lint-action@v2
with:
version: latest
working-directory: agent
args: --timeout=10m
- name: Go lint - shared
uses: golangci/golangci-lint-action@v2
with:

75
Dockerfile
View File

@ -1,75 +0,0 @@
ARG TARGETARCH=amd64
### Base builder image for native builds architecture
FROM golang:1.17-alpine AS builder-native-base
ENV CGO_ENABLED=1 GOOS=linux
RUN apk add --no-cache g++ perl-utils
### Intermediate builder image for x86-64 native builds
FROM builder-native-base AS builder-for-amd64
ENV GOARCH=amd64
ENV BPF_TARGET=amd64 BPF_CFLAGS="-O2 -g -D__TARGET_ARCH_x86"
### Intermediate builder image for AArch64 native builds
FROM builder-native-base AS builder-for-arm64v8
ENV GOARCH=arm64
ENV BPF_TARGET=arm64 BPF_CFLAGS="-O2 -g -D__TARGET_ARCH_arm64"
### Final builder image where the build happens
# Possible build strategies:
# TARGETARCH=amd64
# TARGETARCH=arm64v8
ARG TARGETARCH=amd64
FROM builder-for-${TARGETARCH} AS builder
# Move to agent working directory (/agent-build)
WORKDIR /app/agent-build
COPY agent/go.mod agent/go.sum ./
COPY shared/go.mod shared/go.mod ../shared/
COPY logger/go.mod logger/go.mod ../logger/
RUN go mod download
# Copy and build agent code
COPY shared ../shared
COPY logger ../logger
COPY agent .
ARG COMMIT_HASH
ARG GIT_BRANCH
ARG BUILD_TIMESTAMP
ARG VER=0.0
WORKDIR /app/agent-build
RUN go build -ldflags="-extldflags=-static -s -w \
-X 'github.com/kubeshark/kubeshark/agent/pkg/version.GitCommitHash=${COMMIT_HASH}' \
-X 'github.com/kubeshark/kubeshark/agent/pkg/version.Branch=${GIT_BRANCH}' \
-X 'github.com/kubeshark/kubeshark/agent/pkg/version.BuildTimestamp=${BUILD_TIMESTAMP}' \
-X 'github.com/kubeshark/kubeshark/agent/pkg/version.Ver=${VER}'" -o kubesharkagent .
# Download Basenine executable, verify the sha1sum
ADD https://github.com/up9inc/basenine/releases/download/v0.8.3/basenine_linux_${GOARCH} ./basenine_linux_${GOARCH}
ADD https://github.com/up9inc/basenine/releases/download/v0.8.3/basenine_linux_${GOARCH}.sha256 ./basenine_linux_${GOARCH}.sha256
RUN shasum -a 256 -c basenine_linux_"${GOARCH}".sha256 && \
chmod +x ./basenine_linux_"${GOARCH}" && \
mv ./basenine_linux_"${GOARCH}" ./basenine
### The shipped image
ARG TARGETARCH=amd64
FROM ${TARGETARCH}/busybox:latest
# gin-gonic runs in debug mode without this
ENV GIN_MODE=release
WORKDIR /app/data/
WORKDIR /app
# Copy binary and config files from /build to root folder of scratch container.
COPY --from=builder ["/app/agent-build/kubesharkagent", "."]
COPY --from=builder ["/app/agent-build/basenine", "/usr/local/bin/basenine"]
ENTRYPOINT ["/app/kubesharkagent"]

59
Makefile
View File

@ -8,86 +8,33 @@ SHELL=/bin/bash
# HELP
# This will output the help for each task
# thanks to https://marmelab.com/blog/2016/02/29/auto-documented-makefile.html
.PHONY: help ui agent agent-debug cli docker
.PHONY: help cli
help: ## This help.
@awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST)
.DEFAULT_GOAL := help
# Variables and lists
TS_SUFFIX="$(shell date '+%s')"
GIT_BRANCH="$(shell git branch | grep \* | cut -d ' ' -f2 | tr '[:upper:]' '[:lower:]' | tr '/' '_')"
BUCKET_PATH=static.up9.io/kubeshark/$(GIT_BRANCH)
export VER?=0.0
ui: ## Build UI.
@(cd ui; npm i ; npm run build; )
@ls -l ui/build
cli: ## Build CLI.
@echo "building cli"; cd cli && $(MAKE) build
cli-debug: ## Build CLI.
@echo "building cli"; cd cli && $(MAKE) build-debug
agent: ## Build agent.
@(echo "building kubeshark agent .." )
@(cd agent; go build -o build/kubesharkagent main.go)
@ls -l agent/build
agent-debug: ## Build agent for debug.
@(echo "building kubeshark agent for debug.." )
@(cd agent; go build -gcflags="all=-N -l" -o build/kubesharkagent main.go)
@ls -l agent/build
docker: ## Build and publish agent docker image.
$(MAKE) push-docker
agent-docker: ## Build agent docker image.
@echo "Building agent docker image"
@docker build -t kubeshark/kubeshark:devlatest .
push: push-docker push-cli ## Build and publish agent docker image & CLI.
push-docker: ## Build and publish agent docker image.
@echo "publishing Docker image .. "
devops/build-push-featurebranch.sh
push-cli: ## Build and publish CLI.
@echo "publishing CLI .. "
@cd cli; $(MAKE) build-all
@echo "publishing file ${OUTPUT_FILE} .."
#gsutil mv gs://${BUCKET_PATH}/${OUTPUT_FILE} gs://${BUCKET_PATH}/${OUTPUT_FILE}.${SUFFIX}
gsutil cp -r ./cli/bin/* gs://${BUCKET_PATH}/
gsutil setmeta -r -h "Cache-Control:public, max-age=30" gs://${BUCKET_PATH}/\*
clean: clean-ui clean-agent clean-cli clean-docker ## Clean all build artifacts.
clean-ui: ## Clean UI.
@(rm -rf ui/build ; echo "UI cleanup done" )
clean-agent: ## Clean agent.
@(rm -rf agent/build ; echo "agent cleanup done" )
clean: clean-ui clean-cli ## Clean all build artifacts.
clean-cli: ## Clean CLI.
@(cd cli; make clean ; echo "CLI cleanup done" )
clean-docker: ## Run clean docker
@(echo "DOCKER cleanup - NOT IMPLEMENTED YET " )
lint: ## Run lint on all modules
cd agent && golangci-lint run
cd shared && golangci-lint run
cd cli && golangci-lint run
test: test-cli test-agent test-shared
test: test-cli test-shared
test-cli: ## Run cli tests
@echo "running cli tests"; cd cli && $(MAKE) test
test-agent: ## Run agent tests
@echo "running agent tests"; cd agent && $(MAKE) test
test-shared: ## Run shared tests
@echo "running shared tests"; cd shared && $(MAKE) test

6
agent/.snyk
View File

@ -1,6 +0,0 @@
# Snyk (https://snyk.io) policy file, patches or ignores known vulnerabilities.
version: v1.14.0
ignore:
SNYK-GOLANG-GITHUBCOMGINGONICGIN-1041736:
- '*':
reason: None Given

2
agent/Makefile
View File

@ -1,2 +0,0 @@
test: ## Run agent tests.
@go test ./... -coverpkg=./... -race -coverprofile=coverage.out -covermode=atomic

129
agent/go.mod
View File

@ -1,129 +0,0 @@
module github.com/kubeshark/kubeshark/agent
go 1.17
require (
github.com/antelman107/net-wait-go v0.0.0-20210623112055-cf684aebda7b
github.com/chanced/openapi v0.0.8
github.com/djherbis/atime v1.1.0
github.com/gin-contrib/pprof v1.3.0
github.com/gin-gonic/gin v1.7.7
github.com/go-playground/locales v0.14.0
github.com/go-playground/universal-translator v0.18.0
github.com/go-playground/validator/v10 v10.10.0
github.com/google/uuid v1.3.0
github.com/gorilla/websocket v1.5.0
github.com/jinzhu/copier v0.3.5
github.com/kubeshark/kubeshark/logger v0.0.0
github.com/kubeshark/kubeshark/shared v0.0.0
github.com/kubeshark/worker v0.1.1
github.com/nav-inc/datetime v0.1.3
github.com/op/go-logging v0.0.0-20160315200505-970db520ece7
github.com/orcaman/concurrent-map v1.0.0
github.com/stretchr/testify v1.7.0
github.com/up9inc/basenine/client/go v0.0.0-20220612112747-3b28eeac9c51
github.com/wI2L/jsondiff v0.1.1
k8s.io/api v0.23.3
k8s.io/apimachinery v0.23.3
k8s.io/client-go v0.23.3
)
require (
cloud.google.com/go/compute v1.2.0 // indirect
github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
github.com/Azure/go-autorest v14.2.0+incompatible // indirect
github.com/Azure/go-autorest/autorest v0.11.24 // indirect
github.com/Azure/go-autorest/autorest/adal v0.9.18 // indirect
github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect
github.com/Azure/go-autorest/logger v0.2.1 // indirect
github.com/Azure/go-autorest/tracing v0.6.0 // indirect
github.com/MakeNowJust/heredoc v1.0.0 // indirect
github.com/PuerkitoBio/purell v1.1.1 // indirect
github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5 // indirect
github.com/chanced/dynamic v0.0.0-20211210164248-f8fadb1d735b // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/evanphx/json-patch v5.6.0+incompatible // indirect
github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f // indirect
github.com/fatih/camelcase v1.0.0 // indirect
github.com/frankban/quicktest v1.14.0 // indirect
github.com/fvbommel/sortorder v1.0.2 // indirect
github.com/gin-contrib/sse v0.1.0 // indirect
github.com/go-errors/errors v1.4.2 // indirect
github.com/go-logr/logr v1.2.2 // indirect
github.com/go-openapi/jsonpointer v0.19.5 // indirect
github.com/go-openapi/jsonreference v0.19.6 // indirect
github.com/go-openapi/swag v0.21.1 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang-jwt/jwt/v4 v4.2.0 // indirect
github.com/golang/protobuf v1.5.2 // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/google/btree v1.0.1 // indirect
github.com/google/go-cmp v0.5.7 // indirect
github.com/google/gofuzz v1.2.0 // indirect
github.com/google/martian v2.1.0+incompatible // indirect
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
github.com/googleapis/gnostic v0.5.5 // indirect
github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 // indirect
github.com/imdario/mergo v0.3.12 // indirect
github.com/inconshreveable/mousetrap v1.0.0 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/klauspost/compress v1.14.2 // indirect
github.com/leodido/go-urn v1.2.1 // indirect
github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect
github.com/mailru/easyjson v0.7.7 // indirect
github.com/mattn/go-isatty v0.0.14 // indirect
github.com/mertyildiran/gqlparser/v2 v2.4.6 // indirect
github.com/mitchellh/go-wordwrap v1.0.1 // indirect
github.com/moby/spdystream v0.2.0 // indirect
github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect
github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect
github.com/ohler55/ojg v1.12.12 // indirect
github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
github.com/pierrec/lz4 v2.6.1+incompatible // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/russross/blackfriday v1.6.0 // indirect
github.com/santhosh-tekuri/jsonschema/v5 v5.0.0 // indirect
github.com/segmentio/kafka-go v0.4.27 // indirect
github.com/spf13/cobra v1.3.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/tidwall/gjson v1.14.0 // indirect
github.com/tidwall/match v1.1.1 // indirect
github.com/tidwall/pretty v1.2.0 // indirect
github.com/tidwall/sjson v1.2.4 // indirect
github.com/ugorji/go/codec v1.2.6 // indirect
github.com/xlab/treeprint v1.1.0 // indirect
go.starlark.net v0.0.0-20220203230714-bb14e151c28f // indirect
golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b // indirect
golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd // indirect
golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect
golang.org/x/sys v0.0.0-20220207234003-57398862261d // indirect
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
golang.org/x/text v0.3.7 // indirect
golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/protobuf v1.27.1 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
k8s.io/cli-runtime v0.23.3 // indirect
k8s.io/component-base v0.23.3 // indirect
k8s.io/klog/v2 v2.40.1 // indirect
k8s.io/kube-openapi v0.0.0-20220124234850-424119656bbf // indirect
k8s.io/kubectl v0.23.3 // indirect
k8s.io/utils v0.0.0-20220127004650-9b3446523e65 // indirect
sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2 // indirect
sigs.k8s.io/kustomize/api v0.11.1 // indirect
sigs.k8s.io/kustomize/kyaml v0.13.3 // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect
sigs.k8s.io/yaml v1.3.0 // indirect
)
replace github.com/kubeshark/kubeshark/logger v0.0.0 => ../logger
replace github.com/kubeshark/kubeshark/shared v0.0.0 => ../shared

1299
agent/go.sum
View File

File diff suppressed because it is too large Load Diff

128
agent/main.go
View File

@ -1,128 +0,0 @@
package main
import (
"flag"
"net/http"
"os"
"os/signal"
"github.com/gin-contrib/pprof"
"github.com/gin-gonic/gin"
"github.com/kubeshark/kubeshark/agent/pkg/dependency"
"github.com/kubeshark/kubeshark/agent/pkg/entries"
"github.com/kubeshark/kubeshark/agent/pkg/middlewares"
"github.com/kubeshark/kubeshark/agent/pkg/oas"
"github.com/kubeshark/kubeshark/agent/pkg/routes"
"github.com/kubeshark/kubeshark/agent/pkg/servicemap"
"github.com/kubeshark/kubeshark/agent/pkg/utils"
"github.com/kubeshark/kubeshark/agent/pkg/api"
"github.com/kubeshark/kubeshark/agent/pkg/app"
"github.com/kubeshark/kubeshark/agent/pkg/config"
"github.com/kubeshark/kubeshark/logger"
"github.com/kubeshark/kubeshark/shared"
tapApi "github.com/kubeshark/worker/api"
"github.com/op/go-logging"
)
var namespace = flag.String("namespace", "", "Resolve IPs if they belong to resources in this namespace (default is all)")
var port = flag.Int("port", 80, "Port number of the HTTP server")
var profiler = flag.Bool("profiler", false, "Run pprof server")
func main() {
initializeDependencies()
logLevel := determineLogLevel()
logger.InitLoggerStd(logLevel)
flag.Parse()
app.LoadExtensions()
ginApp := runInApiServerMode(*namespace)
if *profiler {
pprof.Register(ginApp)
}
utils.StartServer(ginApp, *port)
signalChan := make(chan os.Signal, 1)
signal.Notify(signalChan, os.Interrupt)
<-signalChan
logger.Log.Info("Exiting")
}
func hostApi(socketHarOutputChannel chan<- *tapApi.OutputChannelItem) *gin.Engine {
ginApp := gin.Default()
ginApp.GET("/echo", func(c *gin.Context) {
c.JSON(http.StatusOK, "Here is Kubeshark agent")
})
eventHandlers := api.RoutesEventHandlers{
SocketOutChannel: socketHarOutputChannel,
}
ginApp.Use(middlewares.CORSMiddleware())
api.WebSocketRoutes(ginApp, &eventHandlers)
if config.Config.OAS.Enable {
routes.OASRoutes(ginApp)
}
if config.Config.ServiceMap {
routes.ServiceMapRoutes(ginApp)
}
routes.QueryRoutes(ginApp)
routes.EntriesRoutes(ginApp)
routes.MetadataRoutes(ginApp)
routes.StatusRoutes(ginApp)
routes.DbRoutes(ginApp)
routes.ReplayRoutes(ginApp)
return ginApp
}
func runInApiServerMode(namespace string) *gin.Engine {
if err := config.LoadConfig(); err != nil {
logger.Log.Fatalf("Error loading config file %v", err)
}
app.ConfigureBasenineServer(shared.BasenineHost, shared.BaseninePort, config.Config.MaxDBSizeBytes, config.Config.LogLevel, config.Config.InsertionFilter)
api.StartResolving(namespace)
enableExpFeatureIfNeeded()
return hostApi(app.GetEntryInputChannel())
}
func enableExpFeatureIfNeeded() {
if config.Config.OAS.Enable {
oasGenerator := dependency.GetInstance(dependency.OasGeneratorDependency).(oas.OasGenerator)
oasGenerator.Start()
}
if config.Config.ServiceMap {
serviceMapGenerator := dependency.GetInstance(dependency.ServiceMapGeneratorDependency).(servicemap.ServiceMap)
serviceMapGenerator.Enable()
}
}
func determineLogLevel() (logLevel logging.Level) {
logLevel, err := logging.LogLevel(os.Getenv(shared.LogLevelEnvVar))
if err != nil {
logLevel = logging.INFO
}
return
}
func initializeDependencies() {
dependency.RegisterGenerator(dependency.ServiceMapGeneratorDependency, func() interface{} { return servicemap.GetDefaultServiceMapInstance() })
dependency.RegisterGenerator(dependency.OasGeneratorDependency, func() interface{} { return oas.GetDefaultOasGeneratorInstance(config.Config.OAS.MaxExampleLen) })
dependency.RegisterGenerator(dependency.EntriesInserter, func() interface{} { return api.GetBasenineEntryInserterInstance() })
dependency.RegisterGenerator(dependency.EntriesProvider, func() interface{} { return &entries.BasenineEntriesProvider{} })
dependency.RegisterGenerator(dependency.EntriesSocketStreamer, func() interface{} { return &api.BasenineEntryStreamer{} })
dependency.RegisterGenerator(dependency.EntryStreamerSocketConnector, func() interface{} { return &api.DefaultEntryStreamerSocketConnector{} })
}

71
agent/pkg/api/entry_streamer_socket_connector.go
View File

@ -1,71 +0,0 @@
package api
import (
"fmt"
"github.com/kubeshark/kubeshark/agent/pkg/models"
tapApi "github.com/kubeshark/worker/api"
basenine "github.com/up9inc/basenine/client/go"
)
type EntryStreamerSocketConnector interface {
SendEntry(socketId int, entry *tapApi.Entry, params *WebSocketParams) error
SendMetadata(socketId int, metadata *basenine.Metadata) error
SendToastError(socketId int, err error) error
CleanupSocket(socketId int)
}
type DefaultEntryStreamerSocketConnector struct{}
func (e *DefaultEntryStreamerSocketConnector) SendEntry(socketId int, entry *tapApi.Entry, params *WebSocketParams) error {
var message []byte
if params.EnableFullEntries {
message, _ = models.CreateFullEntryWebSocketMessage(entry)
} else {
protocol, ok := protocolsMap[entry.Protocol.ToString()]
if !ok {
return fmt.Errorf("protocol not found, protocol: %v", protocol)
}
extension, ok := extensionsMap[protocol.Name]
if !ok {
return fmt.Errorf("extension not found, extension: %v", protocol.Name)
}
base := extension.Dissector.Summarize(entry)
message, _ = models.CreateBaseEntryWebSocketMessage(base)
}
if err := SendToSocket(socketId, message); err != nil {
return err
}
return nil
}
func (e *DefaultEntryStreamerSocketConnector) SendMetadata(socketId int, metadata *basenine.Metadata) error {
metadataBytes, _ := models.CreateWebsocketQueryMetadataMessage(metadata)
if err := SendToSocket(socketId, metadataBytes); err != nil {
return err
}
return nil
}
func (e *DefaultEntryStreamerSocketConnector) SendToastError(socketId int, err error) error {
toastBytes, _ := models.CreateWebsocketToastMessage(&models.ToastMessage{
Type: "error",
AutoClose: 5000,
Text: fmt.Sprintf("Syntax error: %s", err.Error()),
})
if err := SendToSocket(socketId, toastBytes); err != nil {
return err
}
return nil
}
func (e *DefaultEntryStreamerSocketConnector) CleanupSocket(socketId int) {
socketObj := connectedWebsockets[socketId]
socketCleanup(socketId, socketObj)
}

171
agent/pkg/api/main.go
View File

@ -1,171 +0,0 @@
package api
import (
"bufio"
"context"
"encoding/json"
"fmt"
"os"
"path"
"sort"
"strings"
"time"
"github.com/kubeshark/kubeshark/agent/pkg/dependency"
"github.com/kubeshark/kubeshark/agent/pkg/oas"
"github.com/kubeshark/kubeshark/agent/pkg/servicemap"
"github.com/kubeshark/kubeshark/agent/pkg/har"
"github.com/kubeshark/kubeshark/agent/pkg/holder"
"github.com/kubeshark/kubeshark/agent/pkg/providers"
"github.com/kubeshark/kubeshark/agent/pkg/resolver"
"github.com/kubeshark/kubeshark/agent/pkg/utils"
"github.com/kubeshark/kubeshark/logger"
tapApi "github.com/kubeshark/worker/api"
)
var k8sResolver *resolver.Resolver
func StartResolving(namespace string) {
errOut := make(chan error, 100)
res, err := resolver.NewFromInCluster(errOut, namespace)
if err != nil {
logger.Log.Infof("error creating k8s resolver %s", err)
return
}
ctx := context.Background()
res.Start(ctx)
go func() {
for {
err := <-errOut
logger.Log.Infof("name resolving error %s", err)
}
}()
k8sResolver = res
holder.SetResolver(res)
}
func StartReadingEntries(harChannel <-chan *tapApi.OutputChannelItem, workingDir *string, extensionsMap map[string]*tapApi.Extension) {
if workingDir != nil && *workingDir != "" {
startReadingFiles(*workingDir)
} else {
startReadingChannel(harChannel, extensionsMap)
}
}
func startReadingFiles(workingDir string) {
if err := os.MkdirAll(workingDir, os.ModePerm); err != nil {
logger.Log.Errorf("Failed to make dir: %s, err: %v", workingDir, err)
return
}
for {
dir, _ := os.Open(workingDir)
dirFiles, _ := dir.Readdir(-1)
var harFiles []os.FileInfo
for _, fileInfo := range dirFiles {
if strings.HasSuffix(fileInfo.Name(), ".har") {
harFiles = append(harFiles, fileInfo)
}
}
sort.Sort(utils.ByModTime(harFiles))
if len(harFiles) == 0 {
logger.Log.Infof("Waiting for new files")
time.Sleep(3 * time.Second)
continue
}
fileInfo := harFiles[0]
inputFilePath := path.Join(workingDir, fileInfo.Name())
file, err := os.Open(inputFilePath)
utils.CheckErr(err)
var inputHar har.HAR
decErr := json.NewDecoder(bufio.NewReader(file)).Decode(&inputHar)
utils.CheckErr(decErr)
rmErr := os.Remove(inputFilePath)
utils.CheckErr(rmErr)
}
}
func startReadingChannel(outputItems <-chan *tapApi.OutputChannelItem, extensionsMap map[string]*tapApi.Extension) {
if outputItems == nil {
panic("Channel of captured messages is nil")
}
for item := range outputItems {
extension := extensionsMap[item.Protocol.Name]
resolvedSource, resolvedDestination, namespace := resolveIP(item.ConnectionInfo)
if namespace == "" && item.Namespace != tapApi.UnknownNamespace {
namespace = item.Namespace
}
kubesharkEntry := extension.Dissector.Analyze(item, resolvedSource, resolvedDestination, namespace)
data, err := json.Marshal(kubesharkEntry)
if err != nil {
logger.Log.Errorf("Error while marshaling entry: %v", err)
continue
}
entryInserter := dependency.GetInstance(dependency.EntriesInserter).(EntryInserter)
if err := entryInserter.Insert(kubesharkEntry); err != nil {
logger.Log.Errorf("Error inserting entry, err: %v", err)
}
summary := extension.Dissector.Summarize(kubesharkEntry)
providers.EntryAdded(len(data), summary)
serviceMapGenerator := dependency.GetInstance(dependency.ServiceMapGeneratorDependency).(servicemap.ServiceMapSink)
serviceMapGenerator.NewTCPEntry(kubesharkEntry.Source, kubesharkEntry.Destination, &item.Protocol)
oasGenerator := dependency.GetInstance(dependency.OasGeneratorDependency).(oas.OasGeneratorSink)
oasGenerator.HandleEntry(kubesharkEntry)
}
}
func resolveIP(connectionInfo *tapApi.ConnectionInfo) (resolvedSource string, resolvedDestination string, namespace string) {
if k8sResolver != nil {
unresolvedSource := connectionInfo.ClientIP
resolvedSourceObject := k8sResolver.Resolve(unresolvedSource)
if resolvedSourceObject == nil {
logger.Log.Debugf("Cannot find resolved name to source: %s", unresolvedSource)
if os.Getenv("SKIP_NOT_RESOLVED_SOURCE") == "1" {
return
}
} else {
resolvedSource = resolvedSourceObject.FullAddress
namespace = resolvedSourceObject.Namespace
}
unresolvedDestination := fmt.Sprintf("%s:%s", connectionInfo.ServerIP, connectionInfo.ServerPort)
resolvedDestinationObject := k8sResolver.Resolve(unresolvedDestination)
if resolvedDestinationObject == nil {
logger.Log.Debugf("Cannot find resolved name to dest: %s", unresolvedDestination)
if os.Getenv("SKIP_NOT_RESOLVED_DEST") == "1" {
return
}
} else {
resolvedDestination = resolvedDestinationObject.FullAddress
// Overwrite namespace (if it was set according to the source)
// Only overwrite if non-empty
if resolvedDestinationObject.Namespace != "" {
namespace = resolvedDestinationObject.Namespace
}
}
}
return resolvedSource, resolvedDestination, namespace
}
func CheckIsServiceIP(address string) bool {
if k8sResolver == nil {
return false
}
return k8sResolver.CheckIsServiceIP(address)
}

72
agent/pkg/api/socket_data_inserter.go
View File

@ -1,72 +0,0 @@
package api
import (
"encoding/json"
"fmt"
"sync"
"time"
"github.com/kubeshark/kubeshark/logger"
"github.com/kubeshark/kubeshark/shared"
"github.com/kubeshark/worker/api"
basenine "github.com/up9inc/basenine/client/go"
)
type EntryInserter interface {
Insert(entry *api.Entry) error
}
type BasenineEntryInserter struct {
connection *basenine.Connection
}
var instance *BasenineEntryInserter
var once sync.Once
func GetBasenineEntryInserterInstance() *BasenineEntryInserter {
once.Do(func() {
instance = &BasenineEntryInserter{}
})
return instance
}
func (e *BasenineEntryInserter) Insert(entry *api.Entry) error {
if e.connection == nil {
e.connection = initializeConnection()
}
data, err := json.Marshal(entry)
if err != nil {
return fmt.Errorf("error marshling entry, err: %v", err)
}
if err := e.connection.SendText(string(data)); err != nil {
e.connection.Close()
e.connection = nil
return fmt.Errorf("error sending text to database, err: %v", err)
}
return nil
}
func initializeConnection() *basenine.Connection {
for {
connection, err := basenine.NewConnection(shared.BasenineHost, shared.BaseninePort)
if err != nil {
logger.Log.Errorf("Can't establish a new connection to Basenine server: %v", err)
time.Sleep(shared.BasenineReconnectInterval * time.Second)
continue
}
if err = connection.InsertMode(); err != nil {
logger.Log.Errorf("Insert mode call failed: %v", err)
connection.Close()
time.Sleep(shared.BasenineReconnectInterval * time.Second)
continue
}
return connection
}
}

171
agent/pkg/api/socket_data_streamer.go
View File

@ -1,171 +0,0 @@
package api
import (
"context"
"encoding/json"
"time"
"github.com/kubeshark/kubeshark/agent/pkg/dependency"
"github.com/kubeshark/kubeshark/logger"
"github.com/kubeshark/kubeshark/shared"
tapApi "github.com/kubeshark/worker/api"
basenine "github.com/up9inc/basenine/client/go"
)
type EntryStreamer interface {
Get(ctx context.Context, socketId int, params *WebSocketParams) error
}
type BasenineEntryStreamer struct{}
func (e *BasenineEntryStreamer) Get(ctx context.Context, socketId int, params *WebSocketParams) error {
var connection *basenine.Connection
entryStreamerSocketConnector := dependency.GetInstance(dependency.EntryStreamerSocketConnector).(EntryStreamerSocketConnector)
connection, err := basenine.NewConnection(shared.BasenineHost, shared.BaseninePort)
if err != nil {
logger.Log.Errorf("Failed to establish a connection to Basenine: %v", err)
entryStreamerSocketConnector.CleanupSocket(socketId)
return err
}
data := make(chan []byte)
meta := make(chan []byte)
query := params.Query
if err = basenine.Validate(shared.BasenineHost, shared.BaseninePort, query); err != nil {
if err := entryStreamerSocketConnector.SendToastError(socketId, err); err != nil {
return err
}
entryStreamerSocketConnector.CleanupSocket(socketId)
return err
}
leftOff, err := e.fetch(socketId, params, entryStreamerSocketConnector)
if err != nil {
logger.Log.Errorf("Fetch error: %v", err)
}
handleDataChannel := func(c *basenine.Connection, data chan []byte) {
for {
bytes := <-data
if string(bytes) == basenine.CloseChannel {
return
}
var entry *tapApi.Entry
if err = json.Unmarshal(bytes, &entry); err != nil {
logger.Log.Debugf("Error unmarshalling entry: %v", err)
continue
}
if err := entryStreamerSocketConnector.SendEntry(socketId, entry, params); err != nil {
logger.Log.Errorf("Error sending entry to socket, err: %v", err)
return
}
}
}
handleMetaChannel := func(c *basenine.Connection, meta chan []byte) {
for {
bytes := <-meta
if string(bytes) == basenine.CloseChannel {
return
}
var metadata *basenine.Metadata
if err = json.Unmarshal(bytes, &metadata); err != nil {
logger.Log.Debugf("Error unmarshalling metadata: %v", err)
continue
}
if err := entryStreamerSocketConnector.SendMetadata(socketId, metadata); err != nil {
logger.Log.Errorf("Error sending metadata to socket, err: %v", err)
return
}
}
}
go handleDataChannel(connection, data)
go handleMetaChannel(connection, meta)
if err = connection.Query(leftOff, query, data, meta); err != nil {
logger.Log.Errorf("Query mode call failed: %v", err)
entryStreamerSocketConnector.CleanupSocket(socketId)
return err
}
go func() {
<-ctx.Done()
data <- []byte(basenine.CloseChannel)
meta <- []byte(basenine.CloseChannel)
connection.Close()
}()
return nil
}
// Reverses a []byte slice.
func (e *BasenineEntryStreamer) fetch(socketId int, params *WebSocketParams, connector EntryStreamerSocketConnector) (leftOff string, err error) {
if params.Fetch <= 0 {
leftOff = params.LeftOff
return
}
var data [][]byte
var firstMeta []byte
var lastMeta []byte
data, firstMeta, lastMeta, err = basenine.Fetch(
shared.BasenineHost,
shared.BaseninePort,
params.LeftOff,
-1,
params.Query,
params.Fetch,
time.Duration(params.TimeoutMs)*time.Millisecond,
)
if err != nil {
return
}
var firstMetadata *basenine.Metadata
if err = json.Unmarshal(firstMeta, &firstMetadata); err != nil {
return
}
leftOff = firstMetadata.LeftOff
var lastMetadata *basenine.Metadata
if err = json.Unmarshal(lastMeta, &lastMetadata); err != nil {
return
}
if err = connector.SendMetadata(socketId, lastMetadata); err != nil {
return
}
data = e.reverseBytesSlice(data)
for _, row := range data {
var entry *tapApi.Entry
if err = json.Unmarshal(row, &entry); err != nil {
break
}
if err = connector.SendEntry(socketId, entry, params); err != nil {
return
}
}
return
}
// Reverses a []byte slice.
func (e *BasenineEntryStreamer) reverseBytesSlice(arr [][]byte) (newArr [][]byte) {
for i := len(arr) - 1; i >= 0; i-- {
newArr = append(newArr, arr[i])
}
return newArr
}

166
agent/pkg/api/socket_routes.go
View File

@ -1,166 +0,0 @@
package api
import (
"fmt"
"net/http"
"sync"
"time"
"github.com/gin-gonic/gin"
"github.com/gorilla/websocket"
"github.com/kubeshark/kubeshark/agent/pkg/models"
"github.com/kubeshark/kubeshark/agent/pkg/utils"
"github.com/kubeshark/kubeshark/logger"
tapApi "github.com/kubeshark/worker/api"
)
var (
extensionsMap map[string]*tapApi.Extension // global
protocolsMap map[string]*tapApi.Protocol //global
)
func InitMaps(extensions map[string]*tapApi.Extension, protocols map[string]*tapApi.Protocol) {
extensionsMap = extensions
protocolsMap = protocols
}
type EventHandlers interface {
WebSocketConnect(c *gin.Context, socketId int, isTapper bool)
WebSocketDisconnect(socketId int, isTapper bool)
WebSocketMessage(socketId int, isTapper bool, message []byte)
}
type SocketConnection struct {
connection *websocket.Conn
lock *sync.Mutex
eventHandlers EventHandlers
isTapper bool
}
type WebSocketParams struct {
LeftOff string `json:"leftOff"`
Query string `json:"query"`
EnableFullEntries bool `json:"enableFullEntries"`
Fetch int `json:"fetch"`
TimeoutMs int `json:"timeoutMs"`
}
var (
websocketUpgrader = websocket.Upgrader{
ReadBufferSize: 1024,
WriteBufferSize: 1024,
}
websocketIdsLock = sync.Mutex{}
connectedWebsockets map[int]*SocketConnection
connectedWebsocketIdCounter = 0
SocketGetBrowserHandler gin.HandlerFunc
SocketGetTapperHandler gin.HandlerFunc
)
func init() {
websocketUpgrader.CheckOrigin = func(r *http.Request) bool { return true } // like cors for web socket
connectedWebsockets = make(map[int]*SocketConnection)
}
func WebSocketRoutes(app *gin.Engine, eventHandlers EventHandlers) {
SocketGetBrowserHandler = func(c *gin.Context) {
websocketHandler(c, eventHandlers, false)
}
SocketGetTapperHandler = func(c *gin.Context) {
websocketHandler(c, eventHandlers, true)
}
app.GET("/ws", func(c *gin.Context) {
SocketGetBrowserHandler(c)
})
app.GET("/wsTapper", func(c *gin.Context) { // TODO: add m2m authentication to this route
SocketGetTapperHandler(c)
})
}
func websocketHandler(c *gin.Context, eventHandlers EventHandlers, isTapper bool) {
ws, err := websocketUpgrader.Upgrade(c.Writer, c.Request, nil)
if err != nil {
logger.Log.Errorf("failed to set websocket upgrade: %v", err)
return
}
websocketIdsLock.Lock()
connectedWebsocketIdCounter++
socketId := connectedWebsocketIdCounter
connectedWebsockets[socketId] = &SocketConnection{connection: ws, lock: &sync.Mutex{}, eventHandlers: eventHandlers, isTapper: isTapper}
websocketIdsLock.Unlock()
defer func() {
if socketConnection := connectedWebsockets[socketId]; socketConnection != nil {
socketCleanup(socketId, socketConnection)
}
}()
eventHandlers.WebSocketConnect(c, socketId, isTapper)
startTimeBytes, _ := models.CreateWebsocketStartTimeMessage(utils.StartTime)
if err = SendToSocket(socketId, startTimeBytes); err != nil {
logger.Log.Error(err)
}
for {
_, msg, err := ws.ReadMessage()
if err != nil {
if _, ok := err.(*websocket.CloseError); ok {
logger.Log.Debugf("received websocket close message, socket id: %d", socketId)
} else {
logger.Log.Errorf("error reading message, socket id: %d, error: %v", socketId, err)
}
break
}
eventHandlers.WebSocketMessage(socketId, isTapper, msg)
}
}
func SendToSocket(socketId int, message []byte) error {
socketObj := connectedWebsockets[socketId]
if socketObj == nil {
return fmt.Errorf("socket %v is disconnected", socketId)
}
socketObj.lock.Lock() // gorilla socket panics from concurrent writes to a single socket
defer socketObj.lock.Unlock()
if connectedWebsockets[socketId] == nil {
return fmt.Errorf("socket %v is disconnected", socketId)
}
if err := socketObj.connection.SetWriteDeadline(time.Now().Add(time.Second * 10)); err != nil {
socketCleanup(socketId, socketObj)
return fmt.Errorf("error setting timeout to socket %v, err: %v", socketId, err)
}
if err := socketObj.connection.WriteMessage(websocket.TextMessage, message); err != nil {
socketCleanup(socketId, socketObj)
return fmt.Errorf("failed to write message to socket %v, err: %v", socketId, err)
}
return nil
}
func socketCleanup(socketId int, socketConnection *SocketConnection) {
err := socketConnection.connection.Close()
if err != nil {
logger.Log.Errorf("error closing socket connection for socket id %d: %v", socketId, err)
}
websocketIdsLock.Lock()
connectedWebsockets[socketId] = nil
websocketIdsLock.Unlock()
socketConnection.eventHandlers.WebSocketDisconnect(socketId, socketConnection.isTapper)
}

158
agent/pkg/api/socket_server_handlers.go
View File

@ -1,158 +0,0 @@
package api
import (
"context"
"encoding/json"
"sync"
"github.com/gin-gonic/gin"
"github.com/kubeshark/kubeshark/agent/pkg/dependency"
"github.com/kubeshark/kubeshark/agent/pkg/models"
"github.com/kubeshark/kubeshark/agent/pkg/providers/tappedPods"
"github.com/kubeshark/kubeshark/agent/pkg/providers/tappers"
tapApi "github.com/kubeshark/worker/api"
"github.com/kubeshark/kubeshark/logger"
"github.com/kubeshark/kubeshark/shared"
)
type BrowserClient struct {
dataStreamCancelFunc context.CancelFunc
}
var browserClients = make(map[int]*BrowserClient, 0)
var tapperClientSocketUUIDs = make([]int, 0)
var socketListLock = sync.Mutex{}
type RoutesEventHandlers struct {
EventHandlers
SocketOutChannel chan<- *tapApi.OutputChannelItem
}
func (h *RoutesEventHandlers) WebSocketConnect(_ *gin.Context, socketId int, isTapper bool) {
if isTapper {
logger.Log.Infof("Websocket event - Tapper connected, socket ID: %d", socketId)
tappers.Connected()
socketListLock.Lock()
tapperClientSocketUUIDs = append(tapperClientSocketUUIDs, socketId)
socketListLock.Unlock()
nodeToTappedPodMap := tappedPods.GetNodeToTappedPodMap()
SendTappedPods(socketId, nodeToTappedPodMap)
} else {
logger.Log.Infof("Websocket event - Browser socket connected, socket ID: %d", socketId)
socketListLock.Lock()
browserClients[socketId] = &BrowserClient{}
socketListLock.Unlock()
BroadcastTappedPodsStatus()
}
}
func (h *RoutesEventHandlers) WebSocketDisconnect(socketId int, isTapper bool) {
if isTapper {
logger.Log.Infof("Websocket event - Tapper disconnected, socket ID: %d", socketId)
tappers.Disconnected()
socketListLock.Lock()
removeSocketUUIDFromTapperSlice(socketId)
socketListLock.Unlock()
} else {
logger.Log.Infof("Websocket event - Browser socket disconnected, socket ID: %d", socketId)
socketListLock.Lock()
if browserClients[socketId] != nil && browserClients[socketId].dataStreamCancelFunc != nil {
browserClients[socketId].dataStreamCancelFunc()
}
delete(browserClients, socketId)
socketListLock.Unlock()
}
}
func BroadcastToBrowserClients(message []byte) {
for socketId := range browserClients {
go func(socketId int) {
if err := SendToSocket(socketId, message); err != nil {
logger.Log.Error(err)
}
}(socketId)
}
}
func BroadcastToTapperClients(message []byte) {
for _, socketId := range tapperClientSocketUUIDs {
go func(socketId int) {
if err := SendToSocket(socketId, message); err != nil {
logger.Log.Error(err)
}
}(socketId)
}
}
func (h *RoutesEventHandlers) WebSocketMessage(socketId int, isTapper bool, message []byte) {
if isTapper {
HandleTapperIncomingMessage(message, h.SocketOutChannel, BroadcastToBrowserClients)
} else {
// we initiate the basenine stream after the first websocket message we receive (it contains the entry query), we then store a cancelfunc to later cancel this stream
if browserClients[socketId] != nil && browserClients[socketId].dataStreamCancelFunc == nil {
var params WebSocketParams
if err := json.Unmarshal(message, &params); err != nil {
logger.Log.Errorf("Error: %v", socketId, err)
return
}
entriesStreamer := dependency.GetInstance(dependency.EntriesSocketStreamer).(EntryStreamer)
ctx, cancelFunc := context.WithCancel(context.Background())
err := entriesStreamer.Get(ctx, socketId, &params)
if err != nil {
logger.Log.Errorf("error initializing basenine stream for browser socket %d %+v", socketId, err)
cancelFunc()
} else {
browserClients[socketId].dataStreamCancelFunc = cancelFunc
}
}
}
}
func HandleTapperIncomingMessage(message []byte, socketOutChannel chan<- *tapApi.OutputChannelItem, broadcastMessageFunc func([]byte)) {
var socketMessageBase shared.WebSocketMessageMetadata
err := json.Unmarshal(message, &socketMessageBase)
if err != nil {
logger.Log.Infof("Could not unmarshal websocket message %v", err)
} else {
switch socketMessageBase.MessageType {
case shared.WebSocketMessageTypeTappedEntry:
var tappedEntryMessage models.WebSocketTappedEntryMessage
err := json.Unmarshal(message, &tappedEntryMessage)
if err != nil {
logger.Log.Infof("Could not unmarshal message of message type %s %v", socketMessageBase.MessageType, err)
} else {
// NOTE: This is where the message comes back from the intermediate WebSocket to code.
socketOutChannel <- tappedEntryMessage.Data
}
case shared.WebSocketMessageTypeUpdateStatus:
var statusMessage shared.WebSocketStatusMessage
err := json.Unmarshal(message, &statusMessage)
if err != nil {
logger.Log.Infof("Could not unmarshal message of message type %s %v", socketMessageBase.MessageType, err)
} else {
broadcastMessageFunc(message)
}
default:
logger.Log.Infof("Received socket message of type %s for which no handlers are defined", socketMessageBase.MessageType)
}
}
}
func removeSocketUUIDFromTapperSlice(uuidToRemove int) {
newUUIDSlice := make([]int, 0, len(tapperClientSocketUUIDs))
for _, uuid := range tapperClientSocketUUIDs {
if uuid != uuidToRemove {
newUUIDSlice = append(newUUIDSlice, uuid)
}
}
tapperClientSocketUUIDs = newUUIDSlice
}

40
agent/pkg/api/utils.go
View File

@ -1,40 +0,0 @@
package api
import (
"encoding/json"
"github.com/kubeshark/kubeshark/agent/pkg/providers/tappedPods"
"github.com/kubeshark/kubeshark/logger"
"github.com/kubeshark/kubeshark/shared"
)
func BroadcastTappedPodsStatus() {
tappedPodsStatus := tappedPods.GetTappedPodsStatus()
message := shared.CreateWebSocketStatusMessage(tappedPodsStatus)
if jsonBytes, err := json.Marshal(message); err != nil {
logger.Log.Errorf("Could not Marshal message %v", err)
} else {
BroadcastToBrowserClients(jsonBytes)
}
}
func SendTappedPods(socketId int, nodeToTappedPodMap shared.NodeToPodsMap) {
message := shared.CreateWebSocketTappedPodsMessage(nodeToTappedPodMap)
if jsonBytes, err := json.Marshal(message); err != nil {
logger.Log.Errorf("Could not Marshal message %v", err)
} else {
if err := SendToSocket(socketId, jsonBytes); err != nil {
logger.Log.Error(err)
}
}
}
func BroadcastTappedPodsToTappers(nodeToTappedPodMap shared.NodeToPodsMap) {
message := shared.CreateWebSocketTappedPodsMessage(nodeToTappedPodMap)
if jsonBytes, err := json.Marshal(message); err != nil {
logger.Log.Errorf("Could not Marshal message %v", err)
} else {
BroadcastToTapperClients(jsonBytes)
}
}

135
agent/pkg/app/main.go
View File

@ -1,135 +0,0 @@
package app
import (
"fmt"
"sort"
"time"
"github.com/antelman107/net-wait-go/wait"
"github.com/kubeshark/kubeshark/agent/pkg/api"
"github.com/kubeshark/kubeshark/agent/pkg/providers"
"github.com/kubeshark/kubeshark/agent/pkg/utils"
"github.com/kubeshark/kubeshark/logger"
tapApi "github.com/kubeshark/worker/api"
amqpExt "github.com/kubeshark/worker/extensions/amqp"
httpExt "github.com/kubeshark/worker/extensions/http"
kafkaExt "github.com/kubeshark/worker/extensions/kafka"
redisExt "github.com/kubeshark/worker/extensions/redis"
"github.com/op/go-logging"
basenine "github.com/up9inc/basenine/client/go"
)
var (
Extensions []*tapApi.Extension // global
ExtensionsMap map[string]*tapApi.Extension // global
ProtocolsMap map[string]*tapApi.Protocol //global
)
func LoadExtensions() {
Extensions = make([]*tapApi.Extension, 0)
ExtensionsMap = make(map[string]*tapApi.Extension)
ProtocolsMap = make(map[string]*tapApi.Protocol)
extensionHttp := &tapApi.Extension{}
dissectorHttp := httpExt.NewDissector()
dissectorHttp.Register(extensionHttp)
extensionHttp.Dissector = dissectorHttp
Extensions = append(Extensions, extensionHttp)
ExtensionsMap[extensionHttp.Protocol.Name] = extensionHttp
protocolsHttp := dissectorHttp.GetProtocols()
for k, v := range protocolsHttp {
ProtocolsMap[k] = v
}
extensionAmqp := &tapApi.Extension{}
dissectorAmqp := amqpExt.NewDissector()
dissectorAmqp.Register(extensionAmqp)
extensionAmqp.Dissector = dissectorAmqp
Extensions = append(Extensions, extensionAmqp)
ExtensionsMap[extensionAmqp.Protocol.Name] = extensionAmqp
protocolsAmqp := dissectorAmqp.GetProtocols()
for k, v := range protocolsAmqp {
ProtocolsMap[k] = v
}
extensionKafka := &tapApi.Extension{}
dissectorKafka := kafkaExt.NewDissector()
dissectorKafka.Register(extensionKafka)
extensionKafka.Dissector = dissectorKafka
Extensions = append(Extensions, extensionKafka)
ExtensionsMap[extensionKafka.Protocol.Name] = extensionKafka
protocolsKafka := dissectorKafka.GetProtocols()
for k, v := range protocolsKafka {
ProtocolsMap[k] = v
}
extensionRedis := &tapApi.Extension{}
dissectorRedis := redisExt.NewDissector()
dissectorRedis.Register(extensionRedis)
extensionRedis.Dissector = dissectorRedis
Extensions = append(Extensions, extensionRedis)
ExtensionsMap[extensionRedis.Protocol.Name] = extensionRedis
protocolsRedis := dissectorRedis.GetProtocols()
for k, v := range protocolsRedis {
ProtocolsMap[k] = v
}
sort.Slice(Extensions, func(i, j int) bool {
return Extensions[i].Protocol.Priority < Extensions[j].Protocol.Priority
})
api.InitMaps(ExtensionsMap, ProtocolsMap)
providers.InitProtocolToColor(ProtocolsMap)
}
func ConfigureBasenineServer(host string, port string, dbSize int64, logLevel logging.Level, insertionFilter string) {
if !wait.New(
wait.WithProto("tcp"),
wait.WithWait(200*time.Millisecond),
wait.WithBreak(50*time.Millisecond),
wait.WithDeadline(20*time.Second),
wait.WithDebug(logLevel == logging.DEBUG),
).Do([]string{fmt.Sprintf("%s:%s", host, port)}) {
logger.Log.Panicf("Basenine is not available!")
}
if err := basenine.Limit(host, port, dbSize); err != nil {
logger.Log.Panicf("Error while limiting database size: %v", err)
}
// Define the macros
for _, extension := range Extensions {
macros := extension.Dissector.Macros()
for macro, expanded := range macros {
if err := basenine.Macro(host, port, macro, expanded); err != nil {
logger.Log.Panicf("Error while adding a macro: %v", err)
}
}
}
// Set the insertion filter that comes from the config
if err := basenine.InsertionFilter(host, port, insertionFilter); err != nil {
logger.Log.Errorf("Error while setting the insertion filter: %v", err)
}
utils.StartTime = time.Now().UnixNano() / int64(time.Millisecond)
}
func GetEntryInputChannel() chan *tapApi.OutputChannelItem {
outputItemsChannel := make(chan *tapApi.OutputChannelItem)
filteredOutputItemsChannel := make(chan *tapApi.OutputChannelItem)
go FilterItems(outputItemsChannel, filteredOutputItemsChannel)
go api.StartReadingEntries(filteredOutputItemsChannel, nil, ExtensionsMap)
return outputItemsChannel
}
func FilterItems(inChannel <-chan *tapApi.OutputChannelItem, outChannel chan *tapApi.OutputChannelItem) {
for message := range inChannel {
if message.ConnectionInfo.IsOutgoing && api.CheckIsServiceIP(message.ConnectionInfo.ServerIP) {
continue
}
outChannel <- message
}
}

53
agent/pkg/config/config.go
View File

@ -1,53 +0,0 @@
package config
import (
"encoding/json"
"fmt"
"os"
"github.com/kubeshark/kubeshark/shared"
)
// these values are used when the config.json file is not present
const (
defaultMaxDatabaseSizeBytes int64 = 200 * 1000 * 1000
DefaultDatabasePath string = "./entries"
)
var Config *shared.KubesharkAgentConfig
func LoadConfig() error {
if Config != nil {
return nil
}
filePath := fmt.Sprintf("%s%s", shared.ConfigDirPath, shared.ConfigFileName)
content, err := os.ReadFile(filePath)
if err != nil {
if os.IsNotExist(err) {
return applyDefaultConfig()
}
return err
}
if err = json.Unmarshal(content, &Config); err != nil {
return err
}
return nil
}
func applyDefaultConfig() error {
defaultConfig, err := getDefaultConfig()
if err != nil {
return err
}
Config = defaultConfig
return nil
}
func getDefaultConfig() (*shared.KubesharkAgentConfig, error) {
return &shared.KubesharkAgentConfig{
MaxDBSizeBytes: defaultMaxDatabaseSizeBytes,
AgentDatabasePath: DefaultDatabasePath,
}, nil
}

28
agent/pkg/controllers/db_controller.go
View File

@ -1,28 +0,0 @@
package controllers
import (
"net/http"
"github.com/gin-gonic/gin"
"github.com/kubeshark/kubeshark/agent/pkg/app"
"github.com/kubeshark/kubeshark/agent/pkg/config"
"github.com/kubeshark/kubeshark/shared"
basenine "github.com/up9inc/basenine/client/go"
)
func Flush(c *gin.Context) {
if err := basenine.Flush(shared.BasenineHost, shared.BaseninePort); err != nil {
c.JSON(http.StatusBadRequest, err)
} else {
c.JSON(http.StatusOK, "Flushed.")
}
}
func Reset(c *gin.Context) {
if err := basenine.Reset(shared.BasenineHost, shared.BaseninePort); err != nil {
c.JSON(http.StatusBadRequest, err)
} else {
app.ConfigureBasenineServer(shared.BasenineHost, shared.BaseninePort, config.Config.MaxDBSizeBytes, config.Config.LogLevel, config.Config.InsertionFilter)
c.JSON(http.StatusOK, "Resetted.")
}
}

79
agent/pkg/controllers/entries_controller.go
View File

@ -1,79 +0,0 @@
package controllers
import (
"net/http"
"github.com/kubeshark/kubeshark/agent/pkg/dependency"
"github.com/kubeshark/kubeshark/agent/pkg/entries"
"github.com/kubeshark/kubeshark/agent/pkg/models"
"github.com/kubeshark/kubeshark/agent/pkg/validation"
"github.com/gin-gonic/gin"
"github.com/kubeshark/kubeshark/logger"
)
func HandleEntriesError(c *gin.Context, err error) bool {
if err != nil {
logger.Log.Errorf("Error getting entry: %v", err)
_ = c.Error(err)
c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{
"error": true,
"type": "error",
"autoClose": "5000",
"msg": err.Error(),
})
return true // signal that there was an error and the caller should return
}
return false // no error, can continue
}
func GetEntries(c *gin.Context) {
entriesRequest := &models.EntriesRequest{}
if err := c.BindQuery(entriesRequest); err != nil {
c.JSON(http.StatusBadRequest, err)
}
validationError := validation.Validate(entriesRequest)
if validationError != nil {
c.JSON(http.StatusBadRequest, validationError)
}
if entriesRequest.TimeoutMs == 0 {
entriesRequest.TimeoutMs = 3000
}
entriesProvider := dependency.GetInstance(dependency.EntriesProvider).(entries.EntriesProvider)
entryWrappers, metadata, err := entriesProvider.GetEntries(entriesRequest)
if !HandleEntriesError(c, err) {
baseEntries := make([]interface{}, 0)
for _, entry := range entryWrappers {
baseEntries = append(baseEntries, entry.Base)
}
c.JSON(http.StatusOK, models.EntriesResponse{
Data: baseEntries,
Meta: metadata,
})
}
}
func GetEntry(c *gin.Context) {
singleEntryRequest := &models.SingleEntryRequest{}
if err := c.BindQuery(singleEntryRequest); err != nil {
c.JSON(http.StatusBadRequest, err)
}
validationError := validation.Validate(singleEntryRequest)
if validationError != nil {
c.JSON(http.StatusBadRequest, validationError)
}
id := c.Param("id")
entriesProvider := dependency.GetInstance(dependency.EntriesProvider).(entries.EntriesProvider)
entry, err := entriesProvider.GetEntry(singleEntryRequest, id)
if !HandleEntriesError(c, err) {
c.JSON(http.StatusOK, entry)
}
}

14
agent/pkg/controllers/metadata_controller.go
View File

@ -1,14 +0,0 @@
package controllers
import (
"net/http"
"github.com/gin-gonic/gin"
"github.com/kubeshark/kubeshark/agent/pkg/version"
"github.com/kubeshark/kubeshark/shared"
)
func GetVersion(c *gin.Context) {
resp := shared.VersionResponse{Ver: version.Ver}
c.JSON(http.StatusOK, resp)
}

68
agent/pkg/controllers/oas_controller.go
View File

@ -1,68 +0,0 @@
package controllers
import (
"net/http"
"github.com/chanced/openapi"
"github.com/gin-gonic/gin"
"github.com/kubeshark/kubeshark/agent/pkg/dependency"
"github.com/kubeshark/kubeshark/agent/pkg/oas"
"github.com/kubeshark/kubeshark/logger"
)
func GetOASServers(c *gin.Context) {
m := make([]string, 0)
oasGenerator := dependency.GetInstance(dependency.OasGeneratorDependency).(oas.OasGenerator)
oasGenerator.GetServiceSpecs().Range(func(key, value interface{}) bool {
m = append(m, key.(string))
return true
})
c.JSON(http.StatusOK, m)
}
func GetOASSpec(c *gin.Context) {
oasGenerator := dependency.GetInstance(dependency.OasGeneratorDependency).(oas.OasGenerator)
res, ok := oasGenerator.GetServiceSpecs().Load(c.Param("id"))
if !ok {
c.JSON(http.StatusNotFound, gin.H{
"error": true,
"type": "error",
"autoClose": "5000",
"msg": "Service not found among specs",
})
return // exit
}
gen := res.(*oas.SpecGen)
spec, err := gen.GetSpec()
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{
"error": true,
"type": "error",
"autoClose": "5000",
"msg": err,
})
return // exit
}
c.JSON(http.StatusOK, spec)
}
func GetOASAllSpecs(c *gin.Context) {
res := map[string]*openapi.OpenAPI{}
oasGenerator := dependency.GetInstance(dependency.OasGeneratorDependency).(oas.OasGenerator)
oasGenerator.GetServiceSpecs().Range(func(key, value interface{}) bool {
svc := key.(string)
gen := value.(*oas.SpecGen)
spec, err := gen.GetSpec()
if err != nil {
logger.Log.Warningf("Failed to obtain spec for service %s: %s", svc, err)
return true
}
res[svc] = spec
return true
})
c.JSON(http.StatusOK, res)
}

45
agent/pkg/controllers/oas_controller_test.go
View File

@ -1,45 +0,0 @@
package controllers
import (
"net/http/httptest"
"testing"
"github.com/gin-gonic/gin"
"github.com/kubeshark/kubeshark/agent/pkg/dependency"
"github.com/kubeshark/kubeshark/agent/pkg/oas"
)
func TestGetOASServers(t *testing.T) {
recorder, c := getRecorderAndContext()
GetOASServers(c)
t.Logf("Written body: %s", recorder.Body.String())
}
func TestGetOASAllSpecs(t *testing.T) {
recorder, c := getRecorderAndContext()
GetOASAllSpecs(c)
t.Logf("Written body: %s", recorder.Body.String())
}
func TestGetOASSpec(t *testing.T) {
recorder, c := getRecorderAndContext()
c.Params = []gin.Param{{Key: "id", Value: "some"}}
GetOASSpec(c)
t.Logf("Written body: %s", recorder.Body.String())
}
func getRecorderAndContext() (*httptest.ResponseRecorder, *gin.Context) {
dependency.RegisterGenerator(dependency.OasGeneratorDependency, func() interface{} {
return oas.GetDefaultOasGeneratorInstance(-1)
})
recorder := httptest.NewRecorder()
c, _ := gin.CreateTestContext(recorder)
oas.GetDefaultOasGeneratorInstance(-1).Start()
oas.GetDefaultOasGeneratorInstance(-1).GetServiceSpecs().Store("some", oas.NewGen("some"))
return recorder, c
}

31
agent/pkg/controllers/query_controller.go
View File

@ -1,31 +0,0 @@
package controllers
import (
"net/http"
"github.com/gin-gonic/gin"
"github.com/kubeshark/kubeshark/shared"
basenine "github.com/up9inc/basenine/client/go"
)
type ValidateResponse struct {
Valid bool `json:"valid"`
Message string `json:"message"`
}
func PostValidate(c *gin.Context) {
query := c.PostForm("query")
valid := true
message := ""
err := basenine.Validate(shared.BasenineHost, shared.BaseninePort, query)
if err != nil {
valid = false
message = err.Error()
}
c.JSON(http.StatusOK, ValidateResponse{
Valid: valid,
Message: message,
})
}

34
agent/pkg/controllers/replay_controller.go
View File

@ -1,34 +0,0 @@
package controllers
import (
"net/http"
"time"
"github.com/gin-gonic/gin"
"github.com/kubeshark/kubeshark/agent/pkg/replay"
"github.com/kubeshark/kubeshark/agent/pkg/validation"
"github.com/kubeshark/kubeshark/logger"
)
const (
replayTimeout = 10 * time.Second
)
func ReplayRequest(c *gin.Context) {
logger.Log.Debug("Starting replay")
replayDetails := &replay.Details{}
if err := c.Bind(replayDetails); err != nil {
c.JSON(http.StatusBadRequest, err)
return
}
logger.Log.Debugf("Validating replay, %v", replayDetails)
if err := validation.Validate(replayDetails); err != nil {
c.JSON(http.StatusBadRequest, err)
return
}
logger.Log.Debug("Executing replay, %v", replayDetails)
result := replay.ExecuteRequest(replayDetails, replayTimeout)
c.JSON(http.StatusOK, result)
}

39
agent/pkg/controllers/service_map_controller.go
View File

@ -1,39 +0,0 @@
package controllers
import (
"net/http"
"github.com/kubeshark/kubeshark/agent/pkg/dependency"
"github.com/kubeshark/kubeshark/agent/pkg/servicemap"
"github.com/gin-gonic/gin"
)
type ServiceMapController struct {
service servicemap.ServiceMap
}
func NewServiceMapController() *ServiceMapController {
serviceMapGenerator := dependency.GetInstance(dependency.ServiceMapGeneratorDependency).(servicemap.ServiceMap)
return &ServiceMapController{
service: serviceMapGenerator,
}
}
func (s *ServiceMapController) Status(c *gin.Context) {
c.JSON(http.StatusOK, s.service.GetStatus())
}
func (s *ServiceMapController) Get(c *gin.Context) {
response := &servicemap.ServiceMapResponse{
Status: s.service.GetStatus(),
Nodes: s.service.GetNodes(),
Edges: s.service.GetEdges(),
}
c.JSON(http.StatusOK, response)
}
func (s *ServiceMapController) Reset(c *gin.Context) {
s.service.Reset()
s.Status(c)
}

151
agent/pkg/controllers/service_map_controller_test.go
View File

@ -1,151 +0,0 @@
package controllers
import (
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"testing"
"github.com/kubeshark/kubeshark/agent/pkg/dependency"
"github.com/kubeshark/kubeshark/agent/pkg/servicemap"
"github.com/gin-gonic/gin"
tapApi "github.com/kubeshark/worker/api"
"github.com/stretchr/testify/suite"
)
const (
a = "aService"
b = "bService"
Ip = "127.0.0.1"
Port = "80"
)
var (
TCPEntryA = &tapApi.TCP{
Name: a,
Port: Port,
IP: fmt.Sprintf("%s.%s", Ip, a),
}
TCPEntryB = &tapApi.TCP{
Name: b,
Port: Port,
IP: fmt.Sprintf("%s.%s", Ip, b),
}
)
var ProtocolHttp = &tapApi.Protocol{
ProtocolSummary: tapApi.ProtocolSummary{
Name: "http",
Version: "1.1",
Abbreviation: "HTTP",
},
LongName: "Hypertext Transfer Protocol -- HTTP/1.1",
Macro: "http",
BackgroundColor: "#326de6",
ForegroundColor: "#ffffff",
FontSize: 12,
ReferenceLink: "https://datatracker.ietf.org/doc/html/rfc2616",
Ports: []string{"80", "443", "8080"},
Priority: 0,
}
type ServiceMapControllerSuite struct {
suite.Suite
c *ServiceMapController
w *httptest.ResponseRecorder
g *gin.Context
}
func (s *ServiceMapControllerSuite) SetupTest() {
dependency.RegisterGenerator(dependency.ServiceMapGeneratorDependency, func() interface{} { return servicemap.GetDefaultServiceMapInstance() })
s.c = NewServiceMapController()
s.c.service.Enable()
s.c.service.(servicemap.ServiceMapSink).NewTCPEntry(TCPEntryA, TCPEntryB, ProtocolHttp)
s.w = httptest.NewRecorder()
s.g, _ = gin.CreateTestContext(s.w)
}
func (s *ServiceMapControllerSuite) TestGetStatus() {
assert := s.Assert()
s.c.Status(s.g)
assert.Equal(http.StatusOK, s.w.Code)
var status servicemap.ServiceMapStatus
err := json.Unmarshal(s.w.Body.Bytes(), &status)
assert.NoError(err)
assert.Equal("enabled", status.Status)
assert.Equal(1, status.EntriesProcessedCount)
assert.Equal(2, status.NodeCount)
assert.Equal(1, status.EdgeCount)
}
func (s *ServiceMapControllerSuite) TestGet() {
assert := s.Assert()
s.c.Get(s.g)
assert.Equal(http.StatusOK, s.w.Code)
var response servicemap.ServiceMapResponse
err := json.Unmarshal(s.w.Body.Bytes(), &response)
assert.NoError(err)
// response status
assert.Equal("enabled", response.Status.Status)
assert.Equal(1, response.Status.EntriesProcessedCount)
assert.Equal(2, response.Status.NodeCount)
assert.Equal(1, response.Status.EdgeCount)
// response nodes
aNode := servicemap.ServiceMapNode{
Id: 1,
Name: TCPEntryA.Name,
Entry: TCPEntryA,
Resolved: true,
Count: 1,
}
bNode := servicemap.ServiceMapNode{
Id: 2,
Name: TCPEntryB.Name,
Entry: TCPEntryB,
Resolved: true,
Count: 1,
}
assert.Contains(response.Nodes, aNode)
assert.Contains(response.Nodes, bNode)
assert.Len(response.Nodes, 2)
// response edges
assert.Equal([]servicemap.ServiceMapEdge{
{
Source: aNode,
Destination: bNode,
Protocol: ProtocolHttp,
Count: 1,
},
}, response.Edges)
}
func (s *ServiceMapControllerSuite) TestGetReset() {
assert := s.Assert()
s.c.Reset(s.g)
assert.Equal(http.StatusOK, s.w.Code)
var status servicemap.ServiceMapStatus
err := json.Unmarshal(s.w.Body.Bytes(), &status)
assert.NoError(err)
assert.Equal("enabled", status.Status)
assert.Equal(0, status.EntriesProcessedCount)
assert.Equal(0, status.NodeCount)
assert.Equal(0, status.EdgeCount)
}
func TestServiceMapControllerSuite(t *testing.T) {
suite.Run(t, new(ServiceMapControllerSuite))
}

108
agent/pkg/controllers/status_controller.go
View File

@ -1,108 +0,0 @@
package controllers
import (
"fmt"
"net/http"
"strconv"
"time"
core "k8s.io/api/core/v1"
"github.com/gin-gonic/gin"
"github.com/kubeshark/kubeshark/agent/pkg/api"
"github.com/kubeshark/kubeshark/agent/pkg/holder"
"github.com/kubeshark/kubeshark/agent/pkg/providers"
"github.com/kubeshark/kubeshark/agent/pkg/providers/tappedPods"
"github.com/kubeshark/kubeshark/agent/pkg/providers/tappers"
"github.com/kubeshark/kubeshark/agent/pkg/validation"
"github.com/kubeshark/kubeshark/logger"
"github.com/kubeshark/kubeshark/shared"
"github.com/kubeshark/kubeshark/shared/kubernetes"
)
func HealthCheck(c *gin.Context) {
tappersStatus := make([]*shared.TapperStatus, 0)
for _, value := range tappers.GetStatus() {
tappersStatus = append(tappersStatus, value)
}
response := shared.HealthResponse{
TappedPods: tappedPods.Get(),
ConnectedTappersCount: tappers.GetConnectedCount(),
TappersStatus: tappersStatus,
}
c.JSON(http.StatusOK, response)
}
func PostTappedPods(c *gin.Context) {
var requestTappedPods []core.Pod
if err := c.Bind(&requestTappedPods); err != nil {
c.JSON(http.StatusBadRequest, err)
return
}
podInfos := kubernetes.GetPodInfosForPods(requestTappedPods)
logger.Log.Infof("[Status] POST request: %d tapped pods", len(requestTappedPods))
tappedPods.Set(podInfos)
api.BroadcastTappedPodsStatus()
nodeToTappedPodMap := kubernetes.GetNodeHostToTappedPodsMap(requestTappedPods)
tappedPods.SetNodeToTappedPodMap(nodeToTappedPodMap)
api.BroadcastTappedPodsToTappers(nodeToTappedPodMap)
}
func PostTapperStatus(c *gin.Context) {
tapperStatus := &shared.TapperStatus{}
if err := c.Bind(tapperStatus); err != nil {
c.JSON(http.StatusBadRequest, err)
return
}
if err := validation.Validate(tapperStatus); err != nil {
c.JSON(http.StatusBadRequest, err)
return
}
logger.Log.Infof("[Status] POST request, tapper status: %v", tapperStatus)
tappers.SetStatus(tapperStatus)
api.BroadcastTappedPodsStatus()
}
func GetConnectedTappersCount(c *gin.Context) {
c.JSON(http.StatusOK, tappers.GetConnectedCount())
}
func GetTappingStatus(c *gin.Context) {
tappedPodsStatus := tappedPods.GetTappedPodsStatus()
c.JSON(http.StatusOK, tappedPodsStatus)
}
func GetGeneralStats(c *gin.Context) {
c.JSON(http.StatusOK, providers.GetGeneralStats())
}
func GetTrafficStats(c *gin.Context) {
startTime, endTime, err := getStartEndTime(c)
if err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
return
}
c.JSON(http.StatusOK, providers.GetTrafficStats(startTime, endTime))
}
func getStartEndTime(c *gin.Context) (time.Time, time.Time, error) {
startTimeValue, err := strconv.Atoi(c.Query("startTimeMs"))
if err != nil {
return time.UnixMilli(0), time.UnixMilli(0), fmt.Errorf("invalid start time: %v", err)
}
endTimeValue, err := strconv.Atoi(c.Query("endTimeMs"))
if err != nil {
return time.UnixMilli(0), time.UnixMilli(0), fmt.Errorf("invalid end time: %v", err)
}
return time.UnixMilli(int64(startTimeValue)), time.UnixMilli(int64(endTimeValue)), nil
}
func GetCurrentResolvingInformation(c *gin.Context) {
c.JSON(http.StatusOK, holder.GetResolver().GetMap())
}

11
agent/pkg/dependency/container.go
View File

@ -1,11 +0,0 @@
package dependency
var typeInitializerMap = make(map[ContainerType]func() interface{}, 0)
func RegisterGenerator(name ContainerType, fn func() interface{}) {
typeInitializerMap[name] = fn
}
func GetInstance(name ContainerType) interface{} {
return typeInitializerMap[name]()
}

12
agent/pkg/dependency/type_names.go
View File

@ -1,12 +0,0 @@
package dependency
type ContainerType string
const (
ServiceMapGeneratorDependency ContainerType = "ServiceMapGeneratorDependency"
OasGeneratorDependency ContainerType = "OasGeneratorDependency"
EntriesInserter ContainerType = "EntriesInserter"
EntriesProvider ContainerType = "EntriesProvider"
EntriesSocketStreamer ContainerType = "EntriesSocketStreamer"
EntryStreamerSocketConnector ContainerType = "EntryStreamerSocketConnector"
)

103
agent/pkg/entries/entries_provider.go
View File

@ -1,103 +0,0 @@
package entries
import (
"encoding/json"
"errors"
"fmt"
"time"
"github.com/kubeshark/kubeshark/agent/pkg/app"
"github.com/kubeshark/kubeshark/agent/pkg/models"
"github.com/kubeshark/kubeshark/logger"
"github.com/kubeshark/kubeshark/shared"
tapApi "github.com/kubeshark/worker/api"
basenine "github.com/up9inc/basenine/client/go"
)
type EntriesProvider interface {
GetEntries(entriesRequest *models.EntriesRequest) ([]*tapApi.EntryWrapper, *basenine.Metadata, error)
GetEntry(singleEntryRequest *models.SingleEntryRequest, entryId string) (*tapApi.EntryWrapper, error)
}
type BasenineEntriesProvider struct{}
func (e *BasenineEntriesProvider) GetEntries(entriesRequest *models.EntriesRequest) ([]*tapApi.EntryWrapper, *basenine.Metadata, error) {
data, _, lastMeta, err := basenine.Fetch(shared.BasenineHost, shared.BaseninePort,
entriesRequest.LeftOff, entriesRequest.Direction, entriesRequest.Query,
entriesRequest.Limit, time.Duration(entriesRequest.TimeoutMs)*time.Millisecond)
if err != nil {
return nil, nil, err
}
var dataSlice []*tapApi.EntryWrapper
for _, row := range data {
var entry *tapApi.Entry
err = json.Unmarshal(row, &entry)
if err != nil {
return nil, nil, err
}
protocol, ok := app.ProtocolsMap[entry.Protocol.ToString()]
if !ok {
return nil, nil, fmt.Errorf("protocol not found, protocol: %v", protocol)
}
extension, ok := app.ExtensionsMap[protocol.Name]
if !ok {
return nil, nil, fmt.Errorf("extension not found, extension: %v", protocol.Name)
}
base := extension.Dissector.Summarize(entry)
dataSlice = append(dataSlice, &tapApi.EntryWrapper{
Protocol: *protocol,
Data: entry,
Base: base,
})
}
var metadata *basenine.Metadata
err = json.Unmarshal(lastMeta, &metadata)
if err != nil {
logger.Log.Debugf("Error recieving metadata: %v", err.Error())
}
return dataSlice, metadata, nil
}
func (e *BasenineEntriesProvider) GetEntry(singleEntryRequest *models.SingleEntryRequest, entryId string) (*tapApi.EntryWrapper, error) {
var entry *tapApi.Entry
bytes, err := basenine.Single(shared.BasenineHost, shared.BaseninePort, entryId, singleEntryRequest.Query)
if err != nil {
return nil, err
}
err = json.Unmarshal(bytes, &entry)
if err != nil {
return nil, errors.New(string(bytes))
}
protocol, ok := app.ProtocolsMap[entry.Protocol.ToString()]
if !ok {
return nil, fmt.Errorf("protocol not found, protocol: %v", protocol)
}
extension, ok := app.ExtensionsMap[protocol.Name]
if !ok {
return nil, fmt.Errorf("extension not found, extension: %v", protocol.Name)
}
base := extension.Dissector.Summarize(entry)
var representation []byte
representation, err = extension.Dissector.Represent(entry.Request, entry.Response)
if err != nil {
return nil, err
}
return &tapApi.EntryWrapper{
Protocol: *protocol,
Representation: string(representation),
Data: entry,
Base: base,
}, nil
}

376
agent/pkg/har/types.go
View File

@ -1,376 +0,0 @@
package har
import (
"encoding/base64"
"time"
"unicode/utf8"
"github.com/kubeshark/kubeshark/logger"
)
/*
HTTP Archive (HAR) format
https://w3c.github.io/web-performance/specs/HAR/Overview.html
*/
// HAR is a container type for deserialization
type HAR struct {
Log Log `json:"log"`
}
// Log represents the root of the exported data. This object MUST be present and its name MUST be "log".
type Log struct {
// The object contains the following name/value pairs:
// Required. Version number of the format.
Version string `json:"version"`
// Required. An object of type creator that contains the name and version
// information of the log creator application.
Creator Creator `json:"creator"`
// Optional. An object of type browser that contains the name and version
// information of the user agent.
Browser Browser `json:"browser"`
// Optional. An array of objects of type page, each representing one exported
// (tracked) page. Leave out this field if the application does not support
// grouping by pages.
Pages []Page `json:"pages,omitempty"`
// Required. An array of objects of type entry, each representing one
// exported (tracked) HTTP request.
Entries []Entry `json:"entries"`
// Optional. A comment provided by the user or the application. Sorting
// entries by startedDateTime (starting from the oldest) is preferred way how
// to export data since it can make importing faster. However the reader
// application should always make sure the array is sorted (if required for
// the import).
Comment string `json:"comment"`
}
// Creator contains information about the log creator application
type Creator struct {
// Required. The name of the application that created the log.
Name string `json:"name"`
// Required. The version number of the application that created the log.
Version string `json:"version"`
// Optional. A comment provided by the user or the application.
Comment string `json:"comment,omitempty"`
}
// Browser that created the log
type Browser struct {
// Required. The name of the browser that created the log.
Name string `json:"name"`
// Required. The version number of the browser that created the log.
Version string `json:"version"`
// Optional. A comment provided by the user or the browser.
Comment string `json:"comment"`
}
// Page object for every exported web page and one <entry> object for every HTTP request.
// In case when an HTTP trace tool isn't able to group requests by a page,
// the <pages> object is empty and individual requests doesn't have a parent page.
type Page struct {
/* There is one <page> object for every exported web page and one <entry>
object for every HTTP request. In case when an HTTP trace tool isn't able to
group requests by a page, the <pages> object is empty and individual
requests doesn't have a parent page.
*/
// Date and time stamp for the beginning of the page load
// (ISO 8601 YYYY-MM-DDThh:mm:ss.sTZD, e.g. 2009-07-24T19:20:30.45+01:00).
StartedDateTime string `json:"startedDateTime"`
// Unique identifier of a page within the . Entries use it to refer the parent page.
ID string `json:"id"`
// Page title.
Title string `json:"title"`
// Detailed timing info about page load.
PageTiming PageTiming `json:"pageTiming"`
// (new in 1.2) A comment provided by the user or the application.
Comment string `json:"comment,omitempty"`
}
// PageTiming describes timings for various events (states) fired during the page load.
// All times are specified in milliseconds. If a time info is not available appropriate field is set to -1.
type PageTiming struct {
// Content of the page loaded. Number of milliseconds since page load started
// (page.startedDateTime). Use -1 if the timing does not apply to the current
// request.
// Depeding on the browser, onContentLoad property represents DOMContentLoad
// event or document.readyState == interactive.
OnContentLoad int `json:"onContentLoad"`
// Page is loaded (onLoad event fired). Number of milliseconds since page
// load started (page.startedDateTime). Use -1 if the timing does not apply
// to the current request.
OnLoad int `json:"onLoad"`
// (new in 1.2) A comment provided by the user or the application.
Comment string `json:"comment"`
}
// Entry is a unique, optional Reference to the parent page.
// Leave out this field if the application does not support grouping by pages.
type Entry struct {
Pageref string `json:"pageref,omitempty"`
// Date and time stamp of the request start
// (ISO 8601 YYYY-MM-DDThh:mm:ss.sTZD).
StartedDateTime string `json:"startedDateTime"`
// Total elapsed time of the request in milliseconds. This is the sum of all
// timings available in the timings object (i.e. not including -1 values) .
Time int `json:"time"`
// Detailed info about the request.
Request Request `json:"request"`
// Detailed info about the response.
Response Response `json:"response"`
// Info about cache usage.
Cache Cache `json:"cache"`
// Detailed timing info about request/response round trip.
PageTimings PageTimings `json:"pageTimings"`
// optional (new in 1.2) IP address of the server that was connected
// (result of DNS resolution).
ServerIPAddress string `json:"serverIPAddress,omitempty"`
// optional (new in 1.2) Unique ID of the parent TCP/IP connection, can be
// the client port number. Note that a port number doesn't have to be unique
// identifier in cases where the port is shared for more connections. If the
// port isn't available for the application, any other unique connection ID
// can be used instead (e.g. connection index). Leave out this field if the
// application doesn't support this info.
Connection string `json:"connection,omitempty"`
// (new in 1.2) A comment provided by the user or the application.
Comment string `json:"comment,omitempty"`
}
// Request contains detailed info about performed request.
type Request struct {
// Request method (GET, POST, ...).
Method string `json:"method"`
// Absolute URL of the request (fragments are not included).
URL string `json:"url"`
// Request HTTP Version.
HTTPVersion string `json:"httpVersion"`
// List of cookie objects.
Cookies []Cookie `json:"cookies"`
// List of header objects.
Headers []NVP `json:"headers"`
// List of query parameter objects.
QueryString []NVP `json:"queryString"`
// Posted data.
PostData PostData `json:"postData"`
// Total number of bytes from the start of the HTTP request message until
// (and including) the double CRLF before the body. Set to -1 if the info
// is not available.
HeaderSize int `json:"headerSize"`
// Size of the request body (POST data payload) in bytes. Set to -1 if the
// info is not available.
BodySize int `json:"bodySize"`
// (new in 1.2) A comment provided by the user or the application.
Comment string `json:"comment"`
}
// Response contains detailed info about the response.
type Response struct {
// Response status.
Status int `json:"status"`
// Response status description.
StatusText string `json:"statusText"`
// Response HTTP Version.
HTTPVersion string `json:"httpVersion"`
// List of cookie objects.
Cookies []Cookie `json:"cookies"`
// List of header objects.
Headers []NVP `json:"headers"`
// Details about the response body.
Content Content `json:"content"`
// Redirection target URL from the Location response header.
RedirectURL string `json:"redirectURL"`
// Total number of bytes from the start of the HTTP response message until
// (and including) the double CRLF before the body. Set to -1 if the info is
// not available.
// The size of received response-headers is computed only from headers that
// are really received from the server. Additional headers appended by the
// browser are not included in this number, but they appear in the list of
// header objects.
HeadersSize int `json:"headersSize"`
// Size of the received response body in bytes. Set to zero in case of
// responses coming from the cache (304). Set to -1 if the info is not
// available.
BodySize int `json:"bodySize"`
// optional (new in 1.2) A comment provided by the user or the application.
Comment string `json:"comment,omitempty"`
}
// Cookie contains list of all cookies (used in <request> and <response> objects).
type Cookie struct {
// The name of the cookie.
Name string `json:"name"`
// The cookie value.
Value string `json:"value"`
// optional The path pertaining to the cookie.
Path string `json:"path,omitempty"`
// optional The host of the cookie.
Domain string `json:"domain,omitempty"`
// optional Cookie expiration time.
// (ISO 8601 YYYY-MM-DDThh:mm:ss.sTZD, e.g. 2009-07-24T19:20:30.123+02:00).
Expires string `json:"expires,omitempty"`
// optional Set to true if the cookie is HTTP only, false otherwise.
HTTPOnly bool `json:"httpOnly,omitempty"`
// optional (new in 1.2) True if the cookie was transmitted over ssl, false
// otherwise.
Secure bool `json:"secure,omitempty"`
// optional (new in 1.2) A comment provided by the user or the application.
Comment string `json:"comment,omitempty"`
}
// NVP is simply a name/value pair with a comment
type NVP struct {
Name string `json:"name"`
Value string `json:"value"`
Comment string `json:"comment,omitempty"`
}
// PostData describes posted data, if any (embedded in <request> object).
type PostData struct {
// Mime type of posted data.
MimeType string `json:"mimeType"`
// List of posted parameters (in case of URL encoded parameters).
Params []PostParam `json:"params"`
// Plain text posted data
Text string `json:"text"`
// optional (new in 1.2) A comment provided by the user or the
// application.
Comment string `json:"comment,omitempty"`
}
func (d PostData) B64Decoded() (bool, []byte, string) {
// there is a weird gap in HAR spec 1.2, that does not define encoding for binary POST bodies
// we have own convention of putting `base64` into comment field to handle it similar to response `Content`
return b64Decoded(d.Comment, d.Text)
}
// PostParam is a list of posted parameters, if any (embedded in <postData> object).
type PostParam struct {
// name of a posted parameter.
Name string `json:"name"`
// optional value of a posted parameter or content of a posted file.
Value string `json:"value,omitempty"`
// optional name of a posted file.
FileName string `json:"fileName,omitempty"`
// optional content type of a posted file.
ContentType string `json:"contentType,omitempty"`
// optional (new in 1.2) A comment provided by the user or the application.
Comment string `json:"comment,omitempty"`
}
// Content describes details about response content (embedded in <response> object).
type Content struct {
// Length of the returned content in bytes. Should be equal to
// response.bodySize if there is no compression and bigger when the content
// has been compressed.
Size int `json:"size"`
// optional Number of bytes saved. Leave out this field if the information
// is not available.
Compression int `json:"compression,omitempty"`
// MIME type of the response text (value of the Content-Type response
// header). The charset attribute of the MIME type is included (if
// available).
MimeType string `json:"mimeType"`
// optional Response body sent from the server or loaded from the browser
// cache. This field is populated with textual content only. The text field
// is either HTTP decoded text or a encoded (e.g. "base64") representation of
// the response body. Leave out this field if the information is not
// available.
Text string `json:"text,omitempty"`
// optional (new in 1.2) Encoding used for response text field e.g
// "base64". Leave out this field if the text field is HTTP decoded
// (decompressed & unchunked), than trans-coded from its original character
// set into UTF-8.
Encoding string `json:"encoding,omitempty"`
// optional (new in 1.2) A comment provided by the user or the application.
Comment string `json:"comment,omitempty"`
}
func (c Content) B64Decoded() (bool, []byte, string) {
return b64Decoded(c.Encoding, c.Text)
}
func b64Decoded(enc string, text string) (isBinary bool, asBytes []byte, asString string) {
if enc == "base64" {
decoded, err := base64.StdEncoding.DecodeString(text)
if err != nil {
logger.Log.Warningf("Failed to decode content as base64: %s", text)
return false, []byte(text), text
}
valid := utf8.Valid(decoded)
return !valid, decoded, string(decoded)
} else {
return false, nil, text
}
}
// Cache contains info about a request coming from browser cache.
type Cache struct {
// optional State of a cache entry before the request. Leave out this field
// if the information is not available.
BeforeRequest CacheObject `json:"beforeRequest,omitempty"`
// optional State of a cache entry after the request. Leave out this field if
// the information is not available.
AfterRequest CacheObject `json:"afterRequest,omitempty"`
// optional (new in 1.2) A comment provided by the user or the application.
Comment string `json:"comment,omitempty"`
}
// CacheObject is used by both beforeRequest and afterRequest
type CacheObject struct {
// optional - Expiration time of the cache entry.
Expires string `json:"expires,omitempty"`
// The last time the cache entry was opened.
LastAccess string `json:"lastAccess"`
// Etag
ETag string `json:"eTag"`
// The number of times the cache entry has been opened.
HitCount int `json:"hitCount"`
// optional (new in 1.2) A comment provided by the user or the application.
Comment string `json:"comment,omitempty"`
}
// PageTimings describes various phases within request-response round trip.
// All times are specified in milliseconds.
type PageTimings struct {
Blocked int `json:"blocked,omitempty"`
// optional - Time spent in a queue waiting for a network connection. Use -1
// if the timing does not apply to the current request.
DNS int `json:"dns,omitempty"`
// optional - DNS resolution time. The time required to resolve a host name.
// Use -1 if the timing does not apply to the current request.
Connect int `json:"connect,omitempty"`
// optional - Time required to create TCP connection. Use -1 if the timing
// does not apply to the current request.
Send int `json:"send"`
// Time required to send HTTP request to the server.
Wait int `json:"wait"`
// Waiting for a response from the server.
Receive int `json:"receive"`
// Time required to read entire response from the server (or cache).
Ssl int `json:"ssl,omitempty"`
// optional (new in 1.2) - Time required for SSL/TLS negotiation. If this
// field is defined then the time is also included in the connect field (to
// ensure backward compatibility with HAR 1.1). Use -1 if the timing does not
// apply to the current request.
Comment string `json:"comment,omitempty"`
// optional (new in 1.2) - A comment provided by the user or the application.
}
// TestResult contains results for an individual HTTP request
type TestResult struct {
URL string `json:"url"`
Status int `json:"status"` // 200, 500, etc.
StartTime time.Time `json:"startTime"`
EndTime time.Time `json:"endTime"`
Latency int `json:"latency"` // milliseconds
Method string `json:"method"`
HarFile string `json:"harfile"`
}
// aliases for martian lib compatibility
type Header = NVP
type QueryString = NVP
type Param = PostParam
type Timings = PageTimings

38
agent/pkg/har/types_test.go
View File

@ -1,38 +0,0 @@
package har
import "testing"
func TestContentEncoded(t *testing.T) {
testCases := []struct {
text string
isBinary bool
expectedStr string
binaryLen int
}{
{"not-base64", false, "not-base64", 10},
{"dGVzdA==", false, "test", 4},
{"test", true, "\f@A", 3}, // valid UTF-8 with some non-printable chars
{"IsDggPCAgPiAgID8gICAgN/vv/e/v/u/v7/9v7+/vyIKIu+3kO+3ke+3ku+3k++3lO+3le+3lu+3l++3mO+3me+3mu+3m++3nO+3ne+3nu+3n++3oO+3oe+3ou+3o++3pO+3pe+3pu+3p++3qO+3qe+3qu+3q++3rO+3re+3ru+3ryIK", true, "test", 132}, // invalid UTF-8 (thus binary), taken from https://www.cl.cam.ac.uk/~mgk25/ucs/examples/UTF-8-test.txt
}
for _, tc := range testCases {
c := Content{
Encoding: "base64",
Text: tc.text,
}
isBinary, asBytes, asString := c.B64Decoded()
_ = asBytes
if tc.isBinary != isBinary {
t.Errorf("Binary flag mismatch: %t != %t", tc.isBinary, isBinary)
}
if !isBinary && tc.expectedStr != asString {
t.Errorf("Decode value mismatch: %s != %s", tc.expectedStr, asString)
}
if tc.binaryLen != len(asBytes) {
t.Errorf("Binary len mismatch: %d != %d", tc.binaryLen, len(asBytes))
}
}
}

215
agent/pkg/har/utils.go
View File

@ -1,215 +0,0 @@
package har
import (
"bytes"
"errors"
"fmt"
"strconv"
"strings"
"time"
"github.com/kubeshark/kubeshark/logger"
)
func BuildHeaders(rawHeaders map[string]interface{}) ([]Header, string, string, string, string, string) {
var host, scheme, authority, path, status string
headers := make([]Header, 0, len(rawHeaders))
for key, value := range rawHeaders {
headers = append(headers, Header{
Name: key,
Value: value.(string),
})
if key == "Host" {
host = value.(string)
}
if key == ":authority" {
authority = value.(string)
}
if key == ":scheme" {
scheme = value.(string)
}
if key == ":path" {
path = value.(string)
}
if key == ":status" {
status = value.(string)
}
}
return headers, host, scheme, authority, path, status
}
func BuildPostParams(rawParams []interface{}) []Param {
params := make([]Param, 0, len(rawParams))
for _, param := range rawParams {
p := param.(map[string]interface{})
name := ""
if p["name"] != nil {
name = p["name"].(string)
}
value := ""
if p["value"] != nil {
value = p["value"].(string)
}
fileName := ""
if p["fileName"] != nil {
fileName = p["fileName"].(string)
}
contentType := ""
if p["contentType"] != nil {
contentType = p["contentType"].(string)
}
params = append(params, Param{
Name: name,
Value: value,
FileName: fileName,
ContentType: contentType,
})
}
return params
}
func NewRequest(request map[string]interface{}) (harRequest *Request, err error) {
headers, host, scheme, authority, path, _ := BuildHeaders(request["headers"].(map[string]interface{}))
cookies := make([]Cookie, 0)
postData, _ := request["postData"].(map[string]interface{})
mimeType := postData["mimeType"]
if mimeType == nil {
mimeType = ""
}
text := postData["text"]
postDataText := ""
if text != nil {
postDataText = text.(string)
}
queryString := make([]QueryString, 0)
for key, value := range request["queryString"].(map[string]interface{}) {
if valuesInterface, ok := value.([]interface{}); ok {
for _, valueInterface := range valuesInterface {
queryString = append(queryString, QueryString{
Name: key,
Value: valueInterface.(string),
})
}
} else {
queryString = append(queryString, QueryString{
Name: key,
Value: value.(string),
})
}
}
url := fmt.Sprintf("http://%s%s", host, request["url"].(string))
if strings.HasPrefix(mimeType.(string), "application/grpc") {
url = fmt.Sprintf("%s://%s%s", scheme, authority, path)
}
harParams := make([]Param, 0)
if postData["params"] != nil {
harParams = BuildPostParams(postData["params"].([]interface{}))
}
harRequest = &Request{
Method: request["method"].(string),
URL: url,
HTTPVersion: request["httpVersion"].(string),
HeaderSize: -1,
BodySize: bytes.NewBufferString(postDataText).Len(),
QueryString: queryString,
Headers: headers,
Cookies: cookies,
PostData: PostData{
MimeType: mimeType.(string),
Params: harParams,
Text: postDataText,
},
}
return
}
func NewResponse(response map[string]interface{}) (harResponse *Response, err error) {
headers, _, _, _, _, _status := BuildHeaders(response["headers"].(map[string]interface{}))
cookies := make([]Cookie, 0)
content, _ := response["content"].(map[string]interface{})
mimeType := content["mimeType"]
if mimeType == nil {
mimeType = ""
}
encoding := content["encoding"]
text := content["text"]
bodyText := ""
if text != nil {
bodyText = text.(string)
}
harContent := &Content{
Encoding: encoding.(string),
MimeType: mimeType.(string),
Text: bodyText,
Size: len(bodyText),
}
status := int(response["status"].(float64))
if strings.HasPrefix(mimeType.(string), "application/grpc") {
if _status != "" {
status, err = strconv.Atoi(_status)
}
if err != nil {
logger.Log.Errorf("Failed converting status to int %s (%v,%+v)", err, err, err)
return nil, errors.New("failed converting response status to int for HAR")
}
}
harResponse = &Response{
HTTPVersion: response["httpVersion"].(string),
Status: status,
StatusText: response["statusText"].(string),
HeadersSize: -1,
BodySize: bytes.NewBufferString(bodyText).Len(),
Headers: headers,
Cookies: cookies,
Content: *harContent,
}
return
}
func NewEntry(request map[string]interface{}, response map[string]interface{}, startTime time.Time, elapsedTime int64) (*Entry, error) {
harRequest, err := NewRequest(request)
if err != nil {
logger.Log.Errorf("Failed converting request to HAR %s (%v,%+v)", err, err, err)
return nil, errors.New("failed converting request to HAR")
}
harResponse, err := NewResponse(response)
if err != nil {
logger.Log.Errorf("Failed converting response to HAR %s (%v,%+v)", err, err, err)
return nil, errors.New("failed converting response to HAR")
}
if elapsedTime < 1 {
elapsedTime = 1
}
harEntry := Entry{
StartedDateTime: startTime.Format(time.RFC3339),
Time: int(elapsedTime),
Request: *harRequest,
Response: *harResponse,
Cache: Cache{},
PageTimings: PageTimings{
Send: -1,
Wait: -1,
Receive: int(elapsedTime),
},
}
return &harEntry, nil
}

13
agent/pkg/holder/main.go
View File

@ -1,13 +0,0 @@
package holder
import "github.com/kubeshark/kubeshark/agent/pkg/resolver"
var k8sResolver *resolver.Resolver
func SetResolver(param *resolver.Resolver) {
k8sResolver = param
}
func GetResolver() *resolver.Resolver {
return k8sResolver
}

19
agent/pkg/middlewares/cors.go
View File

@ -1,19 +0,0 @@
package middlewares
import "github.com/gin-gonic/gin"
func CORSMiddleware() gin.HandlerFunc {
return func(c *gin.Context) {
c.Writer.Header().Set("Access-Control-Allow-Origin", "*")
c.Writer.Header().Set("Access-Control-Allow-Credentials", "true")
c.Writer.Header().Set("Access-Control-Allow-Headers", "Content-Type, Content-Length, Accept-Encoding, X-CSRF-Token, Authorization, accept, origin, Cache-Control, X-Requested-With, x-session-token")
c.Writer.Header().Set("Access-Control-Allow-Methods", "POST, OPTIONS, GET, PUT, DELETE")
if c.Request.Method == "OPTIONS" {
c.AbortWithStatus(204)
return
}
c.Next()
}
}

144
agent/pkg/models/models.go
View File

@ -1,144 +0,0 @@
package models
import (
"encoding/json"
"github.com/kubeshark/kubeshark/agent/pkg/har"
tapApi "github.com/kubeshark/worker/api"
"github.com/kubeshark/kubeshark/shared"
basenine "github.com/up9inc/basenine/client/go"
)
type EntriesRequest struct {
LeftOff string `form:"leftOff" validate:"required"`
Direction int `form:"direction" validate:"required,oneof='1' '-1'"`
Query string `form:"query"`
Limit int `form:"limit" validate:"required,min=1"`
TimeoutMs int `form:"timeoutMs" validate:"min=1"`
}
type SingleEntryRequest struct {
Query string `form:"query"`
}
type EntriesResponse struct {
Data []interface{} `json:"data"`
Meta *basenine.Metadata `json:"meta"`
}
type WebSocketEntryMessage struct {
*shared.WebSocketMessageMetadata
Data *tapApi.BaseEntry `json:"data,omitempty"`
}
type WebSocketFullEntryMessage struct {
*shared.WebSocketMessageMetadata
Data *tapApi.Entry `json:"data,omitempty"`
}
type WebSocketTappedEntryMessage struct {
*shared.WebSocketMessageMetadata
Data *tapApi.OutputChannelItem
}
type ToastMessage struct {
Type string `json:"type"`
AutoClose uint `json:"autoClose"`
Text string `json:"text"`
}
type WebSocketToastMessage struct {
*shared.WebSocketMessageMetadata
Data *ToastMessage `json:"data,omitempty"`
}
type WebSocketQueryMetadataMessage struct {
*shared.WebSocketMessageMetadata
Data *basenine.Metadata `json:"data,omitempty"`
}
type WebSocketStartTimeMessage struct {
*shared.WebSocketMessageMetadata
Data int64 `json:"data"`
}
func CreateBaseEntryWebSocketMessage(base *tapApi.BaseEntry) ([]byte, error) {
message := &WebSocketEntryMessage{
WebSocketMessageMetadata: &shared.WebSocketMessageMetadata{
MessageType: shared.WebSocketMessageTypeEntry,
},
Data: base,
}
return json.Marshal(message)
}
func CreateFullEntryWebSocketMessage(entry *tapApi.Entry) ([]byte, error) {
message := &WebSocketFullEntryMessage{
WebSocketMessageMetadata: &shared.WebSocketMessageMetadata{
MessageType: shared.WebSocketMessageTypeFullEntry,
},
Data: entry,
}
return json.Marshal(message)
}
func CreateWebsocketTappedEntryMessage(base *tapApi.OutputChannelItem) ([]byte, error) {
message := &WebSocketTappedEntryMessage{
WebSocketMessageMetadata: &shared.WebSocketMessageMetadata{
MessageType: shared.WebSocketMessageTypeTappedEntry,
},
Data: base,
}
return json.Marshal(message)
}
func CreateWebsocketToastMessage(base *ToastMessage) ([]byte, error) {
message := &WebSocketToastMessage{
WebSocketMessageMetadata: &shared.WebSocketMessageMetadata{
MessageType: shared.WebSocketMessageTypeToast,
},
Data: base,
}
return json.Marshal(message)
}
func CreateWebsocketQueryMetadataMessage(base *basenine.Metadata) ([]byte, error) {
message := &WebSocketQueryMetadataMessage{
WebSocketMessageMetadata: &shared.WebSocketMessageMetadata{
MessageType: shared.WebSocketMessageTypeQueryMetadata,
},
Data: base,
}
return json.Marshal(message)
}
func CreateWebsocketStartTimeMessage(base int64) ([]byte, error) {
message := &WebSocketStartTimeMessage{
WebSocketMessageMetadata: &shared.WebSocketMessageMetadata{
MessageType: shared.WebSocketMessageTypeStartTime,
},
Data: base,
}
return json.Marshal(message)
}
// ExtendedHAR is the top level object of a HAR log.
type ExtendedHAR struct {
Log *ExtendedLog `json:"log"`
}
// ExtendedLog is the HAR HTTP request and response log.
type ExtendedLog struct {
// Version number of the HAR format.
Version string `json:"version"`
// Creator holds information about the log creator application.
Creator *ExtendedCreator `json:"creator"`
// Entries is a list containing requests and responses.
Entries []*har.Entry `json:"entries"`
}
type ExtendedCreator struct {
*har.Creator
Source *string `json:"_source"`
}

120
agent/pkg/oas/counters.go
View File

@ -1,120 +0,0 @@
package oas
import (
"fmt"
"math"
"strings"
"github.com/chanced/openapi"
)
type Counter struct {
Entries int `json:"entries"`
Failures int `json:"failures"`
FirstSeen float64 `json:"firstSeen"`
LastSeen float64 `json:"lastSeen"`
SumRT float64 `json:"sumRT"`
SumDuration float64 `json:"sumDuration"`
}
func (c *Counter) addEntry(ts float64, rt float64, succ bool, dur float64) {
if dur < 0 {
panic("Duration cannot be negative")
}
c.Entries += 1
c.SumRT += rt
c.SumDuration += dur
if !succ {
c.Failures += 1
}
if c.FirstSeen == 0 {
c.FirstSeen = ts
} else {
c.FirstSeen = math.Min(c.FirstSeen, ts)
}
c.LastSeen = math.Max(c.LastSeen, ts)
}
func (c *Counter) addOther(other *Counter) {
c.Entries += other.Entries
c.SumRT += other.SumRT
c.Failures += other.Failures
c.SumDuration += other.SumDuration
if c.FirstSeen == 0 {
c.FirstSeen = other.FirstSeen
} else {
c.FirstSeen = math.Min(c.FirstSeen, other.FirstSeen)
}
c.LastSeen = math.Max(c.LastSeen, other.LastSeen)
}
type CounterMap map[string]*Counter
func (m *CounterMap) addOther(other *CounterMap) {
for src, cmap := range *other {
if existing, ok := (*m)[src]; ok {
existing.addOther(cmap)
} else {
copied := *cmap
(*m)[src] = &copied
}
}
}
func setCounterMsgIfOk(oldStr string, cnt *Counter) string {
tpl := "Kubeshark observed %d entries (%d failed), at %.3f hits/s, average response time is %.3f seconds"
if oldStr == "" || (strings.HasPrefix(oldStr, "Kubeshark ") && strings.HasSuffix(oldStr, " seconds")) {
return fmt.Sprintf(tpl, cnt.Entries, cnt.Failures, cnt.SumDuration/float64(cnt.Entries), cnt.SumRT/float64(cnt.Entries))
}
return oldStr
}
type CounterMaps struct {
counterTotal Counter
counterMapTotal CounterMap
}
func (m *CounterMaps) processOp(opObj *openapi.Operation) error {
if _, ok := opObj.Extensions.Extension(CountersTotal); ok {
counter := new(Counter)
err := opObj.Extensions.DecodeExtension(CountersTotal, counter)
if err != nil {
return err
}
m.counterTotal.addOther(counter)
opObj.Description = setCounterMsgIfOk(opObj.Description, counter)
}
if _, ok := opObj.Extensions.Extension(CountersPerSource); ok {
counterMap := new(CounterMap)
err := opObj.Extensions.DecodeExtension(CountersPerSource, counterMap)
if err != nil {
return err
}
m.counterMapTotal.addOther(counterMap)
}
return nil
}
func (m *CounterMaps) processOas(oas *openapi.OpenAPI) error {
if oas.Extensions == nil {
oas.Extensions = openapi.Extensions{}
}
err := oas.Extensions.SetExtension(CountersTotal, m.counterTotal)
if err != nil {
return err
}
err = oas.Extensions.SetExtension(CountersPerSource, m.counterMapTotal)
if err != nil {
return nil
}
return nil
}

211
agent/pkg/oas/feeder_test.go
View File

@ -1,211 +0,0 @@
package oas
import (
"bufio"
"encoding/json"
"errors"
"fmt"
"io"
"net/url"
"os"
"path/filepath"
"sort"
"strings"
"testing"
"github.com/kubeshark/kubeshark/agent/pkg/har"
"github.com/kubeshark/kubeshark/logger"
)
func getFiles(baseDir string) (result []string, err error) {
result = make([]string, 0)
logger.Log.Infof("Reading files from tree: %s", baseDir)
inputs := []string{baseDir}
// https://yourbasic.org/golang/list-files-in-directory/
visitor := func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if info.Mode()&os.ModeSymlink != 0 {
path, _ = os.Readlink(path)
inputs = append(inputs, path)
return nil
}
ext := strings.ToLower(filepath.Ext(path))
if !info.IsDir() && (ext == ".har" || ext == ".ldjson") {
result = append(result, path)
}
return nil
}
for len(inputs) > 0 {
path := inputs[0]
inputs = inputs[1:]
err = filepath.Walk(path, visitor)
}
sort.SliceStable(result, func(i, j int) bool {
return fileSize(result[i]) < fileSize(result[j])
})
logger.Log.Infof("Got files: %d", len(result))
return result, err
}
func fileSize(fname string) int64 {
fi, err := os.Stat(fname)
if err != nil {
panic(err)
}
return fi.Size()
}
func feedEntries(fromFiles []string, isSync bool, gen *defaultOasGenerator) (count uint, err error) {
badFiles := make([]string, 0)
cnt := uint(0)
for _, file := range fromFiles {
logger.Log.Info("Processing file: " + file)
ext := strings.ToLower(filepath.Ext(file))
eCnt := uint(0)
switch ext {
case ".har":
eCnt, err = feedFromHAR(file, isSync, gen)
if err != nil {
logger.Log.Warning("Failed processing file: " + err.Error())
badFiles = append(badFiles, file)
continue
}
case ".ldjson":
eCnt, err = feedFromLDJSON(file, isSync, gen)
if err != nil {
logger.Log.Warning("Failed processing file: " + err.Error())
badFiles = append(badFiles, file)
continue
}
default:
return 0, errors.New("Unsupported file extension: " + ext)
}
cnt += eCnt
}
for _, f := range badFiles {
logger.Log.Infof("Bad file: %s", f)
}
return cnt, nil
}
func feedFromHAR(file string, isSync bool, gen *defaultOasGenerator) (uint, error) {
fd, err := os.Open(file)
if err != nil {
panic(err)
}
defer fd.Close()
data, err := io.ReadAll(fd)
if err != nil {
return 0, err
}
var harDoc har.HAR
err = json.Unmarshal(data, &harDoc)
if err != nil {
return 0, err
}
cnt := uint(0)
for _, entry := range harDoc.Log.Entries {
cnt += 1
feedEntry(&entry, "", file, gen, fmt.Sprintf("%024d", cnt))
}
return cnt, nil
}
func feedEntry(entry *har.Entry, source string, file string, gen *defaultOasGenerator, cnt string) {
entry.Comment = file
if entry.Response.Status == 302 {
logger.Log.Debugf("Dropped traffic entry due to permanent redirect status: %s", entry.StartedDateTime)
}
if strings.Contains(entry.Request.URL, "some") { // for debugging
logger.Log.Debugf("Interesting: %s", entry.Request.URL)
}
u, err := url.Parse(entry.Request.URL)
if err != nil {
logger.Log.Errorf("Failed to parse entry URL: %v, err: %v", entry.Request.URL, err)
}
ews := EntryWithSource{Entry: *entry, Source: source, Destination: u.Host, Id: cnt}
gen.handleHARWithSource(&ews)
}
func feedFromLDJSON(file string, isSync bool, gen *defaultOasGenerator) (uint, error) {
fd, err := os.Open(file)
if err != nil {
panic(err)
}
defer fd.Close()
reader := bufio.NewReader(fd)
var meta map[string]interface{}
buf := strings.Builder{}
cnt := uint(0)
source := ""
for {
substr, isPrefix, err := reader.ReadLine()
if err == io.EOF {
break
}
buf.WriteString(string(substr))
if isPrefix {
continue
}
line := buf.String()
buf.Reset()
if meta == nil {
err := json.Unmarshal([]byte(line), &meta)
if err != nil {
return 0, err
}
if s, ok := meta["_source"]; ok && s != nil {
source = s.(string)
}
} else {
var entry har.Entry
err := json.Unmarshal([]byte(line), &entry)
if err != nil {
logger.Log.Warningf("Failed decoding entry: %s", line)
} else {
cnt += 1
feedEntry(&entry, source, file, gen, fmt.Sprintf("%024d", cnt))
}
}
}
return cnt, nil
}
func TestFilesList(t *testing.T) {
res, err := getFiles("./test_artifacts/")
t.Log(len(res))
t.Log(res)
if err != nil || len(res) != 3 {
t.Logf("Should return 2 files but returned %d", len(res))
t.FailNow()
}
}

185
agent/pkg/oas/gibberish.go
View File

@ -1,185 +0,0 @@
package oas
import (
"math"
"regexp"
"strings"
"unicode"
)
var (
patUuid4 = regexp.MustCompile(`(?i)[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}`)
patEmail = regexp.MustCompile(`^\w+([-+.']\w+)*@\w+([-.]\w+)*\.\w+([-.]\w+)*$`)
patLongNum = regexp.MustCompile(`^\d{3,}$`)
patLongNumB = regexp.MustCompile(`[^\d]\d{3,}`)
patLongNumA = regexp.MustCompile(`\d{3,}[^\d]`)
)
func IsGibberish(str string) bool {
if IsVersionString(str) {
return false
}
if patEmail.MatchString(str) {
return true
}
if patUuid4.MatchString(str) {
return true
}
if patLongNum.MatchString(str) || patLongNumB.MatchString(str) || patLongNumA.MatchString(str) {
return true
}
//alNum := cleanStr(str, isAlNumRune)
//alpha := cleanStr(str, isAlphaRune)
// noiseAll := isNoisy(alNum)
//triAll := isTrigramBad(strings.ToLower(alpha))
// _ = noiseAll
isNotAlNum := func(r rune) bool { return !isAlNumRune(r) }
chunks := strings.FieldsFunc(str, isNotAlNum)
noisyLen := 0
alnumLen := 0
for _, chunk := range chunks {
alnumLen += len(chunk)
noise := isNoisy(chunk)
tri := isTrigramBad(strings.ToLower(chunk))
if noise || tri {
noisyLen += len(chunk)
}
}
return float64(noisyLen) > 0
//if float64(noisyLen) > 0 {
// return true
//}
//if len(chunks) > 0 && float64(noisyLen) >= float64(alnumLen)/3.0 {
// return true
//}
//if triAll {
//return true
//}
// return false
}
func noiseLevel(str string) (score float64) {
// opinionated algo of certain char pairs marking the non-human strings
prev := *new(rune)
cnt := 0.0
for _, char := range str {
cnt += 1
if prev > 0 {
switch {
// continued class of upper/lower/digit adds no noise
case unicode.IsUpper(prev) && unicode.IsUpper(char):
case unicode.IsLower(prev) && unicode.IsLower(char):
case unicode.IsDigit(prev) && unicode.IsDigit(char):
// upper =>
case unicode.IsUpper(prev) && unicode.IsLower(char):
score += 0.10
case unicode.IsUpper(prev) && unicode.IsDigit(char):
score += 0.5
// lower =>
case unicode.IsLower(prev) && unicode.IsUpper(char):
score += 0.75
case unicode.IsLower(prev) && unicode.IsDigit(char):
score += 0.5
// digit =>
case unicode.IsDigit(prev) && unicode.IsUpper(char):
score += 0.75
case unicode.IsDigit(prev) && unicode.IsLower(char):
score += 1.0
// the rest is 100% noise
default:
score += 1
}
}
prev = char
}
return score
}
func IsVersionString(component string) bool {
if component == "" {
return false
}
hasV := false
if strings.HasPrefix(component, "v") {
component = component[1:]
hasV = true
}
for _, c := range component {
if string(c) != "." && !unicode.IsDigit(c) {
return false
}
}
if !hasV && !strings.Contains(component, ".") {
return false
}
return true
}
func trigramScore(str string) (float64, int) {
tgScore := 0.0
trigrams := ngrams(str, 3)
if len(trigrams) > 0 {
for _, trigram := range trigrams {
score, found := corpus_trigrams[trigram]
if found {
tgScore += score
}
}
}
return tgScore, len(trigrams)
}
func isTrigramBad(s string) bool {
tgScore, cnt := trigramScore(s)
if cnt > 0 {
val := math.Sqrt(tgScore) / float64(cnt)
val2 := tgScore / float64(cnt)
threshold := 0.005
bad := val < threshold
threshold2 := math.Log(float64(cnt)-2) * 0.1
bad2 := val2 < threshold2
return bad && bad2 // TODO: improve this logic to be clearer
}
return false
}
func isNoisy(s string) bool {
noise := noiseLevel(s)
if len(s) > 0 {
val := (noise * noise) / float64(len(s))
threshold := 0.6
bad := val > threshold
return bad
}
return false
}
func ngrams(s string, n int) []string {
result := make([]string, 0)
for i := 0; i < len(s)-n+1; i++ {
result = append(result, s[i:i+n])
}
return result
}

212
agent/pkg/oas/gibberish_test.go
View File

@ -1,212 +0,0 @@
package oas
import (
"testing"
)
func TestNegative(t *testing.T) {
cases := []string{
"",
"{}",
"0.0.29",
"0.1",
"1.0",
"1.0.0",
"2.1.73",
"abTestV2",
"actionText,setName,setAttribute,save,ignore,onEnd,getContext,end,get",
"AddUserGroupLink",
"advert-management.adBlockerMessage.html",
"agents.author.1.json",
"animated-gif",
"b", // can be valid hexadecimal
"big-danger-coronavirus-panic-greater-crisis",
"breakout-box",
"callback",
"core.algorithm_execution.view",
"core.devices.view",
"data.json",
"dialog.overlay.infinity.json",
"domain-input",
"embeddable", // lul, it's a valid HEX!
"embeddable_blip",
"E PLURIBUS UNUM",
"etc",
"eu-central-1a",
"fcgi-bin",
"footer.include.html",
"fullHashes:find",
"generate-feed",
"GetAds",
"GetCart",
"GetUniversalVariableUser",
"github-audit-exports",
"g.js",
"g.pixel",
".html",
"Hugo Michiels",
"image.sbix",
"index.html",
"iPad",
"Joanna Mazewski",
"LibGit2Sharp",
"Michael_Vaughan1.png",
"New RSS feed has been generated",
"nick-clegg",
"opt-out",
"pixel_details.html",
"post.json",
"profile-method-info",
"project-id",
"publisha.1.json",
"publish_and_moderate",
"Ronna McDaniel",
"rtb-h",
"runs",
"sign-up",
"some-uuid-maybe",
"stable-4.0-version.json",
"StartUpCheckout",
"Steve Flunk",
"sync_a9",
"Ted Cruz",
"test.png",
"token",
"ToList",
"v2.1.3",
"VersionCheck.php",
"v Rusiji",
"Walnut St",
"web_widget",
"zoom_in.cur",
"xray",
"web",
"vipbets1",
"trcc",
"fbpixel",
// TODO below
// "tcfv2",
// "Matt-cartoon-255x206px-small.png",
// "TheTelegraph_portal_white-320-small.png",
// "testdata-10kB.js",
}
for _, str := range cases {
if IsGibberish(str) {
t.Errorf("Mistakenly true: %s", str)
}
}
}
func TestPositive(t *testing.T) {
cases := []string{
"0a0d0174-b338-4520-a1c3-24f7e3d5ec50.html",
"1024807212418223",
"11ca096cbc224a67360493d44a9903",
"1553183382779",
"1554507871",
"19180481",
"203ef0f713abcebd8d62c35c0e3f12f87d71e5e4",
"456795af-b48f-4a8d-9b37-3e932622c2f0",
"601a2bdcc5b69137248ddbbf",
"60fe9aaeaefe2400012df94f",
"610bc3fd5a77a7fa25033fb0",
"610bd0315a77a7fa25034368",
"610bd0315a77a7fa25034368zh",
"6120c057c7a97b03f6986f1b",
"710a462e",
"730970532670-compute@developer.gserviceaccount.com",
"819db2242a648b305395537022523d65",
"952bea17-3776-11ea-9341-42010a84012a",
"a3226860758.html",
"AAAA028295945",
"arn-aws-ecs-eu-west-2-396248696294-cluster-london-01-ECSCluster-27iuIYva8nO4",
"arn-aws-ecs-eu-west-2-396248696294-cluster-london-01-ECSCluster-27iuIYva8nO4", // ?
"bnjksfd897345nl098asd53412kl98",
"c738338322370b47a79251f7510dd", // prefixed hex
"ci12NC01YzkyNTEzYzllMDRhLTAtYy5tb25pdG9yaW5nLmpzb24=", // long base64
"css/login.0f48c49a34eb53ea4623.min.css",
"d_fLLxlhzDilixeBEimaZ5",
"e21f7112-3d3b-4632-9da3-a4af2e0e9166",
"e8782afc112720300c049ff124434b79",
"fb6cjraf9cejut2a",
"i-0236530c66ed02200",
"JEHJW4BKVFDRTMTUQLHKK5WVAU",
"john.dow.1981@protonmail.com",
"MDEyOk9yZ2FuaXphdGlvbjU3MzI0Nzk1",
"MNUTGVFMGLEMFTBH0XSE5E02F6J2DS",
"n63nd45qsj",
"n9z9QGNiz",
"NC4WTmcy",
"proxy.3d2100fd7107262ecb55ce6847f01fa5.html",
"QgAAAC6zw0qH2DJtnXe8Z7rUJP0FgAFKkOhcHdFWzL1ZYggtwBgiB3LSoele9o3ZqFh7iCBhHbVLAnMuJ0HF8hEw7UKecE6wd-MBXgeRMdubGydhAMZSmuUjRpqplML40bmrb8VjJKNZswD1Cg",
"QgAAAC6zw0qH2DJtnXe8Z7rUJP0rG4sjLa_KVLlww5WEDJ__30J15en-K_6Y68jb_rU93e2TFY6fb0MYiQ1UrLNMQufqODHZUl39Lo6cXAOVOThjAMZSmuVH7n85JOYSCgzpvowMAVueGG0Xxg",
"qwerqwerasdfqwer@protonmai.com",
"r-ext-5579e00a95c90",
"r-ext-5579e8b12f11e",
"r-v4-5c92513c9e04a",
"r-v4-5c92513c9e04a-0-c.monitoring.json",
"segments-1563566437171.639994",
"sp_ANQXRpqH_urn$3Auri$3Abase64$3A6698b0a3-97ad-52ce-8fc3-17d99e37a726",
"sp_dxJTfx11_576742227280287872",
"sp_NnUPB5wj_601a2bdcc5b69137248ddbbf",
"sp_NxITuoE4_premiumchron-article-14302157_c_ryGQBs_r_yIWvwP",
"t_52d94268-8810-4a7e-ba87-ffd657a6752f",
"timeouts-1563566437171.639994",
"u_YPF3GsGKMo02",
"0000000000 65535 f",
"0000000178 00000 n",
"0-10000",
"01526123,",
"0,18168,183955,3,4,1151616,5663,731,223,5104,207,3204,10,1051,175,364,1435,4,60,576,241,383,246,5,1102",
"05/10/2020",
"14336456724940333",
"fb6cjraf9cejut2a",
"JEHJW4BKVFDRTMTUQLHKK5WVAU",
// TODO
/*
"0,20",
"0.001",
"YISAtiX1",
"Fxvd1timk", // questionable
"B4GCSkORAJs",
"D_4EDAqenHQ",
"EICJp29EGOk",
"Fxvd1timk",
"GTqMZELYfQQ",
"GZPTpLPEGmwHGWPC",
"_HChnE9NDPY",
"NwhjgIWHgGg",
"production/tsbqksph4xswqjexfbec",
"p/u/bguhrxupr23mw3nwxcrw",
"nRSNapbJZnc",
"zgfpbtolciznub5egzxk",
"zufnu7aimadua9wrgwwo",
"zznto1jzch9yjsbtbrul",
*/
}
for _, str := range cases {
if !IsGibberish(str) {
t.Errorf("Mistakenly false: %s", str)
}
}
}
func TestVersionStrings(t *testing.T) {
cases := []string{
"1.0",
"1.0.0",
"v2.1.3",
"2.1.73",
}
for _, str := range cases {
if !IsVersionString(str) {
t.Errorf("Mistakenly false: %s", str)
}
}
}

77
agent/pkg/oas/ignores.go
View File

@ -1,77 +0,0 @@
package oas
import "strings"
var ignoredExtensions = []string{"gif", "svg", "css", "png", "ico", "js", "woff2", "woff", "jpg", "jpeg", "swf", "ttf", "map", "webp", "otf", "mp3"}
var ignoredCtypePrefixes = []string{"image/", "font/", "video/", "audio/", "text/javascript"}
var ignoredCtypes = []string{"application/javascript", "application/x-javascript", "text/css", "application/font-woff2", "application/font-woff", "application/x-font-woff"}
var ignoredHeaders = []string{
"a-im", "accept",
"authorization", "cache-control", "connection", "content-encoding", "content-length", "content-range", "content-type", "cookie",
"date", "dnt", "expect", "forwarded", "from", "front-end-https", "host", "http2-settings",
"max-forwards", "origin", "pragma", "proxy-authorization", "proxy-connection", "range", "referer",
"save-data", "te", "trailer", "transfer-encoding", "upgrade", "upgrade-insecure-requests", "x-download-options",
"server", "user-agent", "via", "warning", "strict-transport-security", "x-permitted-cross-domain-policies",
"x-att-deviceid", "x-correlation-id", "correlation-id", "x-client-data", "x-dns-prefetch-control",
"x-http-method-override", "x-real-ip", "x-request-id", "x-request-start", "x-requested-with", "x-uidh",
"x-same-domain", "x-content-type-options", "x-frame-options", "x-xss-protection",
"x-wap-profile", "x-scheme", "status", "x-cache", "x-application-context", "retry-after",
"newrelic", "x-cloud-trace-context", "sentry-trace", "x-cache-hits", "x-served-by", "x-span-name",
"expires", "set-cookie", "p3p", "content-security-policy", "content-security-policy-report-only",
"last-modified", "content-language", "x-varnish", "true-client-ip", "akamai-origin-hop",
"keep-alive", "etag", "alt-svc", "x-csrf-token", "x-ua-compatible", "vary", "x-powered-by",
"age", "allow", "www-authenticate", "expect-ct", "timing-allow-origin", "referrer-policy",
"x-aspnet-version", "x-aspnetmvc-version", "x-timer", "x-abuse-info", "x-mod-pagespeed",
"duration_ms",
}
var ignoredHeaderPrefixes = []string{
":", "accept-", "access-control-", "if-", "sec-", "grpc-",
"x-forwarded-", "x-original-", "cf-",
"x-up9-", "x-envoy-", "x-hasura-", "x-b3-", "x-datadog-", "x-envoy-", "x-amz-", "x-newrelic-", "x-prometheus-",
"x-akamai-", "x-spotim-", "x-amzn-", "x-ratelimit-", "x-goog-",
}
func isCtypeIgnored(ctype string) bool {
for _, prefix := range ignoredCtypePrefixes {
if strings.HasPrefix(ctype, prefix) {
return true
}
}
for _, toIgnore := range ignoredCtypes {
if ctype == toIgnore {
return true
}
}
return false
}
func isExtIgnored(path string) bool {
for _, extIgn := range ignoredExtensions {
if strings.HasSuffix(path, "."+extIgn) {
return true
}
}
return false
}
func isHeaderIgnored(name string) bool {
name = strings.ToLower(name)
for _, ignore := range ignoredHeaders {
if name == ignore {
return true
}
}
for _, prefix := range ignoredHeaderPrefixes {
if strings.HasPrefix(name, prefix) {
return true
}
}
return false
}

96
agent/pkg/oas/oas-samples/carts.json
View File

@ -1,96 +0,0 @@
{
"openapi": "3.1.0",
"info": {
"title": "http://carts",
"description": "Kubeshark observed 3 entries (0 failed), at 2.287 hits/s, average response time is 0.017 seconds",
"version": "1.0"
},
"servers": [
{
"url": "http://carts"
}
],
"paths": {
"/carts/{cartId}/items": {
"get": {
"summary": "/carts/{cartId}/items",
"description": "Kubeshark observed 3 entries (0 failed), at 2.287 hits/s, average response time is 0.017 seconds",
"operationId": "84c9b926-1f73-4ab4-b381-3c124528959f",
"responses": {
"200": {
"description": "Successful call with status 200",
"content": {
"application/json": {
"example": [
{
"id": "60fe98fb86c0fc000869a90c",
"itemId": "3395a43e-2d88-40de-b95f-e00e1502085b",
"quantity": 1,
"unitPrice": 18
}
],
"x-sample-entry": "000000000000000000000010"
}
},
"x-sample-entry": "000000000000000000000010"
}
},
"x-counters-per-source": {
"some-source": {
"entries": 3,
"failures": 0,
"firstSeen": 1627298058.3798368,
"lastSeen": 1627298065.2397773,
"sumRT": 0.05,
"sumDuration": 6.859940528869629
}
},
"x-counters-total": {
"entries": 3,
"failures": 0,
"firstSeen": 1627298058.3798368,
"lastSeen": 1627298065.2397773,
"sumRT": 0.05,
"sumDuration": 6.859940528869629
},
"x-last-seen-ts": 1627298065.2397773,
"x-sample-entry": "000000000000000000000010"
},
"parameters": [
{
"name": "cartId",
"in": "path",
"required": true,
"style": "simple",
"schema": {
"type": "string"
},
"examples": {
"example #0": {
"value": "mHK0P7zTktmV1zv57iWAvCTd43FFMHap"
}
},
"x-sample-entry": "000000000000000000000010"
}
]
}
},
"x-counters-per-source": {
"some-source": {
"entries": 3,
"failures": 0,
"firstSeen": 1627298058.3798368,
"lastSeen": 1627298065.2397773,
"sumRT": 0.05,
"sumDuration": 6.859940528869629
}
},
"x-counters-total": {
"entries": 3,
"failures": 0,
"firstSeen": 1627298058.3798368,
"lastSeen": 1627298065.2397773,
"sumRT": 0.05,
"sumDuration": 6.859940528869629
}
}

485
agent/pkg/oas/oas-samples/catalogue.json
View File

@ -1,485 +0,0 @@
{
"openapi": "3.1.0",
"info": {
"title": "Preloaded",
"description": "Test file for loading pre-existing OAS",
"version": "0.1"
},
"paths": {
"/catalogue": {
"get": {
"tags": [
"catalogue"
],
"summary": "/catalogue",
"description": "Kubeshark observed 3 entries (0 failed), at 2.647 hits/s, average response time is 0.008 seconds",
"operationId": "dd6c3dbe-6b6b-4ddd-baed-757e237ddb8a",
"parameters": [
{
"name": "page",
"in": "query",
"required": false,
"style": "form",
"schema": {
"type": "string"
},
"examples": {
"example #0": {
"value": "1"
}
},
"x-sample-entry": "000000000000000000000002"
},
{
"name": "size",
"in": "query",
"required": true,
"style": "form",
"schema": {
"type": "string"
},
"examples": {
"example #0": {
"value": "6"
},
"example #1": {
"value": "3"
},
"example #2": {
"value": "5"
}
},
"x-sample-entry": "000000000000000000000011"
},
{
"name": "tags",
"in": "query",
"required": false,
"style": "form",
"schema": {
"type": "string"
},
"examples": {
"example #0": {
"value": ""
},
"example #1": {
"value": "blue"
}
},
"x-sample-entry": "000000000000000000000007"
},
{
"name": "sort",
"in": "query",
"required": false,
"style": "form",
"schema": {
"type": "string"
},
"examples": {
"example #0": {
"value": "id"
}
},
"x-sample-entry": "000000000000000000000007"
}
],
"responses": {
"200": {
"description": "Successful call with status 200",
"content": {
"application/json": {
"example": [
{
"count": 1,
"description": "Socks fit for a Messiah. You too can experience walking in water with these special edition beauties. Each hole is lovingly proggled to leave smooth edges. The only sock approved by a higher power.",
"id": "03fef6ac-1896-4ce8-bd69-b798f85c6e0b",
"imageUrl": [
"/catalogue/images/holy_1.jpeg",
"/catalogue/images/holy_2.jpeg"
],
"name": "Holy",
"price": 99.99,
"tag": [
"action",
"magic"
]
},
{
"count": 438,
"description": "proident occaecat irure et excepteur labore minim nisi amet irure",
"id": "3395a43e-2d88-40de-b95f-e00e1502085b",
"imageUrl": [
"/catalogue/images/colourful_socks.jpg",
"/catalogue/images/colourful_socks.jpg"
],
"name": "Colourful",
"price": 18,
"tag": [
"brown",
"blue"
]
},
{
"count": 820,
"description": "Ready for action. Engineers: be ready to smash that next bug! Be ready, with these super-action-sport-masterpieces. This particular engineer was chased away from the office with a stick.",
"id": "510a0d7e-8e83-4193-b483-e27e09ddc34d",
"imageUrl": [
"/catalogue/images/puma_1.jpeg",
"/catalogue/images/puma_2.jpeg"
],
"name": "SuperSport XL",
"price": 15,
"tag": [
"sport",
"formal",
"black"
]
},
{
"count": 738,
"description": "A mature sock, crossed, with an air of nonchalance.",
"id": "808a2de1-1aaa-4c25-a9b9-6612e8f29a38",
"imageUrl": [
"/catalogue/images/cross_1.jpeg",
"/catalogue/images/cross_2.jpeg"
],
"name": "Crossed",
"price": 17.32,
"tag": [
"blue",
"action",
"red",
"formal"
]
},
{
"count": 808,
"description": "enim officia aliqua excepteur esse deserunt quis aliquip nostrud anim",
"id": "819e1fbf-8b7e-4f6d-811f-693534916a8b",
"imageUrl": [
"/catalogue/images/WAT.jpg",
"/catalogue/images/WAT2.jpg"
],
"name": "Figueroa",
"price": 14,
"tag": [
"green",
"formal",
"blue"
]
},
{
"count": 175,
"description": "consequat amet cupidatat minim laborum tempor elit ex consequat in",
"id": "837ab141-399e-4c1f-9abc-bace40296bac",
"imageUrl": [
"/catalogue/images/catsocks.jpg",
"/catalogue/images/catsocks2.jpg"
],
"name": "Cat socks",
"price": 15,
"tag": [
"brown",
"formal",
"green"
]
}
],
"x-sample-entry": "000000000000000000000011"
}
},
"x-sample-entry": "000000000000000000000011"
}
},
"x-counters-per-source": {
"some-source": {
"entries": 3,
"failures": 0,
"firstSeen": 1627298057.7849188,
"lastSeen": 1627298065.7258668,
"sumRT": 0.024999999999999998,
"sumDuration": 7.940948009490967
}
},
"x-counters-total": {
"entries": 3,
"failures": 0,
"firstSeen": 1627298057.7849188,
"lastSeen": 1627298065.7258668,
"sumRT": 0.024999999999999998,
"sumDuration": 7.940948009490967
},
"x-last-seen-ts": 1627298065.7258668,
"x-sample-entry": "000000000000000000000011"
}
},
"/catalogue/size": {
"get": {
"tags": [
"catalogue"
],
"summary": "/catalogue/size",
"description": "Kubeshark observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.013 seconds",
"operationId": "2315e69d-9d66-48cf-b3d3-fec9c30bd28b",
"parameters": [
{
"name": "tags",
"in": "query",
"required": true,
"style": "form",
"schema": {
"type": "string"
},
"examples": {
"example #0": {
"value": ""
}
},
"x-sample-entry": "000000000000000000000001"
},
{
"name": "x-some",
"in": "header",
"required": true,
"style": "simple",
"schema": {
"type": "string"
},
"examples": {
"example #0": {
"value": "demo val"
}
},
"x-sample-entry": "000000000000000000000001"
}
],
"responses": {
"200": {
"description": "Successful call with status 200",
"content": {
"application/json": {
"example": {
"err": null,
"size": 9
},
"x-sample-entry": "000000000000000000000001"
}
},
"x-sample-entry": "000000000000000000000001"
}
},
"x-counters-per-source": {
"some-source": {
"entries": 1,
"failures": 0,
"firstSeen": 1627298057.7841518,
"lastSeen": 1627298057.7841518,
"sumRT": 0.013,
"sumDuration": 0
}
},
"x-counters-total": {
"entries": 1,
"failures": 0,
"firstSeen": 1627298057.7841518,
"lastSeen": 1627298057.7841518,
"sumRT": 0.013,
"sumDuration": 0
},
"x-last-seen-ts": 1627298057.7841518,
"x-sample-entry": "000000000000000000000001"
}
},
"/catalogue/{id}": {
"get": {
"tags": [
"catalogue"
],
"summary": "/catalogue/{id}",
"description": "Kubeshark observed 4 entries (0 failed), at 1.899 hits/s, average response time is 0.003 seconds",
"parameters": [
{
"name": "non-required-header",
"in": "header",
"required": false,
"style": "simple",
"schema": {
"type": "string"
},
"example": "some-uuid-maybe"
},
{
"name": "x-some",
"in": "header",
"required": false,
"style": "simple",
"schema": {
"type": "string"
},
"examples": {
"example #0": {
"value": "demoval"
}
},
"x-sample-entry": "000000000000000000000004"
}
],
"responses": {
"200": {
"description": "Successful call with status 200",
"content": {
"application/json": {
"example": {
"count": 438,
"description": "proident occaecat irure et excepteur labore minim nisi amet irure",
"id": "3395a43e-2d88-40de-b95f-e00e1502085b",
"imageUrl": [
"/catalogue/images/colourful_socks.jpg",
"/catalogue/images/colourful_socks.jpg"
],
"name": "Colourful",
"price": 18,
"tag": [
"brown",
"blue"
]
},
"x-sample-entry": "000000000000000000000012"
}
},
"x-sample-entry": "000000000000000000000012"
}
},
"x-counters-per-source": {
"some-source": {
"entries": 4,
"failures": 0,
"firstSeen": 1627298058.1315014,
"lastSeen": 1627298065.7293031,
"sumRT": 0.013999999999999999,
"sumDuration": 7.597801685333252
}
},
"x-counters-total": {
"entries": 4,
"failures": 0,
"firstSeen": 1627298058.1315014,
"lastSeen": 1627298065.7293031,
"sumRT": 0.013999999999999999,
"sumDuration": 7.597801685333252
},
"x-last-seen-ts": 1627298065.7293031,
"x-sample-entry": "000000000000000000000012"
},
"parameters": [
{
"name": "id",
"in": "path",
"required": true,
"style": "simple",
"schema": {
"type": "string"
},
"examples": {
"example #0": {
"value": "3395a43e-2d88-40de-b95f-e00e1502085b"
},
"example #1": {
"value": "808a2de1-1aaa-4c25-a9b9-6612e8f29a38"
}
},
"example": "some-uuid-maybe",
"x-sample-entry": "000000000000000000000012"
}
]
},
"/catalogue/{id}/details": {
"parameters": [
{
"name": "id",
"in": "path",
"required": true,
"style": "simple",
"schema": {
"type": "string"
},
"example": "some-uuid-maybe"
}
]
},
"/tags": {
"get": {
"summary": "/tags",
"description": "Kubeshark observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.007 seconds",
"operationId": "c4d7d0ed-1a78-4370-a049-efe3abc631a6",
"responses": {
"200": {
"description": "Successful call with status 200",
"content": {
"application/json": {
"example": {
"err": null,
"tags": [
"brown",
"geek",
"formal",
"blue",
"skin",
"red",
"action",
"sport",
"black",
"magic",
"green"
]
},
"x-sample-entry": "000000000000000000000003"
}
},
"x-sample-entry": "000000000000000000000003"
}
},
"x-counters-per-source": {
"some-source": {
"entries": 1,
"failures": 0,
"firstSeen": 1627298057.7841816,
"lastSeen": 1627298057.7841816,
"sumRT": 0.007,
"sumDuration": 0
}
},
"x-counters-total": {
"entries": 1,
"failures": 0,
"firstSeen": 1627298057.7841816,
"lastSeen": 1627298057.7841816,
"sumRT": 0.007,
"sumDuration": 0
},
"x-last-seen-ts": 1627298057.7841816,
"x-sample-entry": "000000000000000000000003"
}
}
},
"x-counters-per-source": {
"some-source": {
"entries": 9,
"failures": 0,
"firstSeen": 1627298057.7841518,
"lastSeen": 1627298065.7293031,
"sumRT": 0.05899999999999999,
"sumDuration": 15.538749694824219
}
},
"x-counters-total": {
"entries": 9,
"failures": 0,
"firstSeen": 1627298057.7841518,
"lastSeen": 1627298065.7293031,
"sumRT": 0.05899999999999999,
"sumDuration": 15.538749694824219
}
}

897
agent/pkg/oas/oas-samples/httpbin.org.json
View File

@ -1,897 +0,0 @@
{
"openapi": "3.1.0",
"info": {
"title": "https://httpbin.org",
"description": "Kubeshark observed 19 entries (0 failed), at 0.106 hits/s, average response time is 0.172 seconds",
"version": "1.0"
},
"servers": [
{
"url": "https://httpbin.org"
}
],
"paths": {
"/appears-once": {
"get": {
"summary": "/appears-once",
"description": "Kubeshark observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.630 seconds",
"operationId": "2d34623e-fde8-4720-8390-9a7439051755",
"responses": {
"200": {
"description": "Successful call with status 200",
"content": {
"application/json": {
"example": null,
"x-sample-entry": "000000000000000000000004"
}
},
"x-sample-entry": "000000000000000000000004"
}
},
"x-counters-per-source": {
"": {
"entries": 1,
"failures": 0,
"firstSeen": 1567750580.0471218,
"lastSeen": 1567750580.0471218,
"sumRT": 0.63,
"sumDuration": 0
}
},
"x-counters-total": {
"entries": 1,
"failures": 0,
"firstSeen": 1567750580.0471218,
"lastSeen": 1567750580.0471218,
"sumRT": 0.63,
"sumDuration": 0
},
"x-last-seen-ts": 1567750580.0471218,
"x-sample-entry": "000000000000000000000004"
}
},
"/appears-twice": {
"get": {
"summary": "/appears-twice",
"description": "Kubeshark observed 2 entries (0 failed), at 0.500 hits/s, average response time is 0.630 seconds",
"operationId": "9c5330f3-8062-468b-b5a3-df1ad82b4846",
"responses": {
"200": {
"description": "Successful call with status 200",
"content": {
"application/json": {
"example": null,
"x-sample-entry": "000000000000000000000006"
}
},
"x-sample-entry": "000000000000000000000006"
}
},
"x-counters-per-source": {
"": {
"entries": 2,
"failures": 0,
"firstSeen": 1567750580.7471218,
"lastSeen": 1567750581.7471218,
"sumRT": 1.26,
"sumDuration": 1
}
},
"x-counters-total": {
"entries": 2,
"failures": 0,
"firstSeen": 1567750580.7471218,
"lastSeen": 1567750581.7471218,
"sumRT": 1.26,
"sumDuration": 1
},
"x-last-seen-ts": 1567750581.7471218,
"x-sample-entry": "000000000000000000000006"
}
},
"/body-optional": {
"post": {
"summary": "/body-optional",
"description": "Kubeshark observed 3 entries (0 failed), at 0.003 hits/s, average response time is 0.001 seconds",
"operationId": "34f3d66c-b1f7-4dca-9cab-987fcc8ae472",
"responses": {
"200": {
"description": "Successful call with status 200",
"content": {
"": {
"x-sample-entry": "000000000000000000000012"
}
},
"x-sample-entry": "000000000000000000000012"
}
},
"x-counters-per-source": {
"": {
"entries": 3,
"failures": 0,
"firstSeen": 1567750581.7471218,
"lastSeen": 1567750581.757122,
"sumRT": 0.003,
"sumDuration": 0.010000228881835938
}
},
"x-counters-total": {
"entries": 3,
"failures": 0,
"firstSeen": 1567750581.7471218,
"lastSeen": 1567750581.757122,
"sumRT": 0.003,
"sumDuration": 0.010000228881835938
},
"x-last-seen-ts": 1567750581.757122,
"x-sample-entry": "000000000000000000000012",
"requestBody": {
"description": "Generic request body",
"content": {
"application/json": {
"example": "{\"key\", \"val\"}",
"x-sample-entry": "000000000000000000000011"
}
},
"x-sample-entry": "000000000000000000000012"
}
}
},
"/body-required": {
"post": {
"summary": "/body-required",
"description": "Kubeshark observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.001 seconds",
"operationId": "ff6add53-ab1c-4d4e-b590-0835fa318276",
"responses": {
"200": {
"description": "Successful call with status 200",
"content": {
"": {
"x-sample-entry": "000000000000000000000013"
}
},
"x-sample-entry": "000000000000000000000013"
}
},
"x-counters-per-source": {
"": {
"entries": 1,
"failures": 0,
"firstSeen": 1567750581.757122,
"lastSeen": 1567750581.757122,
"sumRT": 0.001,
"sumDuration": 0
}
},
"x-counters-total": {
"entries": 1,
"failures": 0,
"firstSeen": 1567750581.757122,
"lastSeen": 1567750581.757122,
"sumRT": 0.001,
"sumDuration": 0
},
"x-last-seen-ts": 1567750581.757122,
"x-sample-entry": "000000000000000000000013",
"requestBody": {
"description": "Generic request body",
"content": {
"": {
"example": "body exists",
"x-sample-entry": "000000000000000000000013"
}
},
"required": true,
"x-sample-entry": "000000000000000000000013"
}
}
},
"/form-multipart": {
"post": {
"summary": "/form-multipart",
"description": "Kubeshark observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.001 seconds",
"operationId": "153f0925-9fc7-4e9f-9d33-f1470f25f0f7",
"responses": {
"200": {
"description": "Successful call with status 200",
"content": {
"": {
"example": {},
"x-sample-entry": "000000000000000000000009"
}
},
"x-sample-entry": "000000000000000000000009"
}
},
"x-counters-per-source": {
"": {
"entries": 1,
"failures": 0,
"firstSeen": 1567750582.7471218,
"lastSeen": 1567750582.7471218,
"sumRT": 0.001,
"sumDuration": 0
}
},
"x-counters-total": {
"entries": 1,
"failures": 0,
"firstSeen": 1567750582.7471218,
"lastSeen": 1567750582.7471218,
"sumRT": 0.001,
"sumDuration": 0
},
"x-last-seen-ts": 1567750582.7471218,
"x-sample-entry": "000000000000000000000009",
"requestBody": {
"description": "Generic request body",
"content": {
"multipart/form-data": {
"schema": {
"type": "object",
"required": [
"file",
"path"
],
"properties": {
"file": {
"type": "string",
"contentMediaType": "application/json",
"examples": [
"{\"functions\": 123}"
]
},
"path": {
"type": "string",
"examples": [
"/content/components"
]
}
}
},
"example": "--BOUNDARY\r\nContent-Disposition: form-data; name=\"file\"; filename=\"metadata.json\"\r\nContent-Type: application/json\r\n\r\n{\"functions\": 123}\r\n--BOUNDARY\r\nContent-Disposition: form-data; name=\"path\"\r\n\r\n/content/components\r\n--BOUNDARY--\r\n",
"x-sample-entry": "000000000000000000000009"
}
},
"required": true,
"x-sample-entry": "000000000000000000000009"
}
}
},
"/form-urlencoded": {
"post": {
"summary": "/form-urlencoded",
"description": "Kubeshark observed 2 entries (0 failed), at 0.500 hits/s, average response time is 0.001 seconds",
"operationId": "c92189f5-5636-46eb-ac71-92b17941a568",
"responses": {
"200": {
"description": "Successful call with status 200",
"content": {
"": {
"x-sample-entry": "000000000000000000000008"
}
},
"x-sample-entry": "000000000000000000000008"
}
},
"x-counters-per-source": {
"": {
"entries": 2,
"failures": 0,
"firstSeen": 1567750580.7471218,
"lastSeen": 1567750581.7471218,
"sumRT": 0.002,
"sumDuration": 1
}
},
"x-counters-total": {
"entries": 2,
"failures": 0,
"firstSeen": 1567750580.7471218,
"lastSeen": 1567750581.7471218,
"sumRT": 0.002,
"sumDuration": 1
},
"x-last-seen-ts": 1567750581.7471218,
"x-sample-entry": "000000000000000000000008",
"requestBody": {
"description": "Generic request body",
"content": {
"application/x-www-form-urlencoded": {
"schema": {
"type": "object",
"required": [
"agent-id",
"callback-url",
"token"
],
"properties": {
"agent-id": {
"type": "string",
"examples": [
"ade"
]
},
"callback-url": {
"type": "string",
"examples": [
""
]
},
"optional": {
"type": "string",
"examples": [
"another"
]
},
"token": {
"type": "string",
"examples": [
"sometoken",
"sometoken-second-val"
]
}
}
},
"example": "agent-id=ade\u0026callback-url=\u0026token=sometoken",
"x-sample-entry": "000000000000000000000008"
}
},
"required": true,
"x-sample-entry": "000000000000000000000008"
}
}
},
"/param-patterns/prefix-gibberish-fine/{prefixgibberishfineId}": {
"get": {
"tags": [
"param-patterns"
],
"summary": "/param-patterns/prefix-gibberish-fine/{prefixgibberishfineId}",
"description": "Kubeshark observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.001 seconds",
"operationId": "85270437-7aae-4a5b-b988-3662092463d0",
"responses": {
"200": {
"description": "Successful call with status 200",
"content": {
"": {
"x-sample-entry": "000000000000000000000014"
}
},
"x-sample-entry": "000000000000000000000014"
}
},
"x-counters-per-source": {
"": {
"entries": 1,
"failures": 0,
"firstSeen": 1567750582,
"lastSeen": 1567750582,
"sumRT": 0.001,
"sumDuration": 0
}
},
"x-counters-total": {
"entries": 1,
"failures": 0,
"firstSeen": 1567750582,
"lastSeen": 1567750582,
"sumRT": 0.001,
"sumDuration": 0
},
"x-last-seen-ts": 1567750582,
"x-sample-entry": "000000000000000000000014"
},
"parameters": [
{
"name": "prefixgibberishfineId",
"in": "path",
"required": true,
"style": "simple",
"schema": {
"type": "string"
},
"examples": {
"example #0": {
"value": "234324"
}
},
"x-sample-entry": "000000000000000000000014"
}
]
},
"/param-patterns/{parampatternId}": {
"get": {
"tags": [
"param-patterns"
],
"summary": "/param-patterns/{parampatternId}",
"description": "Kubeshark observed 2 entries (0 failed), at 0.000 hits/s, average response time is 0.001 seconds",
"operationId": "da597734-1cf5-4d3b-917b-6b02dacf7b7b",
"responses": {
"200": {
"description": "Successful call with status 200",
"content": {
"": {
"x-sample-entry": "000000000000000000000018"
}
},
"x-sample-entry": "000000000000000000000018"
}
},
"x-counters-per-source": {
"": {
"entries": 2,
"failures": 0,
"firstSeen": 1567750582.000003,
"lastSeen": 1567750582.000004,
"sumRT": 0.002,
"sumDuration": 9.5367431640625e-7
}
},
"x-counters-total": {
"entries": 2,
"failures": 0,
"firstSeen": 1567750582.000003,
"lastSeen": 1567750582.000004,
"sumRT": 0.002,
"sumDuration": 9.5367431640625e-7
},
"x-last-seen-ts": 1567750582.000004,
"x-sample-entry": "000000000000000000000018"
},
"parameters": [
{
"name": "parampatternId",
"in": "path",
"required": true,
"style": "simple",
"schema": {
"type": "string",
"pattern": "^prefix-gibberish-.+"
},
"examples": {
"example #0": {
"value": "prefix-gibberish-sfdlasdfkadf87sd93284q24r"
},
"example #1": {
"value": "prefix-gibberish-adslkfasdf89sa7dfasddafa8a98sd7kansdf"
},
"example #2": {
"value": "prefix-gibberish-4jk5l2345h2452l4352435jlk45"
},
"example #3": {
"value": "prefix-gibberish-84395h2j4k35hj243j5h2kl34h54k"
},
"example #4": {
"value": "prefix-gibberish-afterwards"
}
},
"x-sample-entry": "000000000000000000000019"
}
]
},
"/param-patterns/{parampatternId}/1": {
"get": {
"tags": [
"param-patterns"
],
"summary": "/param-patterns/{parampatternId}/1",
"description": "Kubeshark observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.001 seconds",
"operationId": "e965a245-9cfc-48ed-94e1-f765eadb3960",
"responses": {
"200": {
"description": "Successful call with status 200",
"content": {
"": {
"x-sample-entry": "000000000000000000000015"
}
},
"x-sample-entry": "000000000000000000000015"
}
},
"x-counters-per-source": {
"": {
"entries": 1,
"failures": 0,
"firstSeen": 1567750582.000001,
"lastSeen": 1567750582.000001,
"sumRT": 0.001,
"sumDuration": 0
}
},
"x-counters-total": {
"entries": 1,
"failures": 0,
"firstSeen": 1567750582.000001,
"lastSeen": 1567750582.000001,
"sumRT": 0.001,
"sumDuration": 0
},
"x-last-seen-ts": 1567750582.000001,
"x-sample-entry": "000000000000000000000015"
},
"parameters": [
{
"name": "parampatternId",
"in": "path",
"required": true,
"style": "simple",
"schema": {
"type": "string",
"pattern": "^prefix-gibberish-.+"
},
"examples": {
"example #0": {
"value": "prefix-gibberish-sfdlasdfkadf87sd93284q24r"
},
"example #1": {
"value": "prefix-gibberish-adslkfasdf89sa7dfasddafa8a98sd7kansdf"
},
"example #2": {
"value": "prefix-gibberish-4jk5l2345h2452l4352435jlk45"
},
"example #3": {
"value": "prefix-gibberish-84395h2j4k35hj243j5h2kl34h54k"
},
"example #4": {
"value": "prefix-gibberish-afterwards"
}
},
"x-sample-entry": "000000000000000000000019"
}
]
},
"/param-patterns/{parampatternId}/static": {
"get": {
"tags": [
"param-patterns"
],
"summary": "/param-patterns/{parampatternId}/static",
"description": "Kubeshark observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.001 seconds",
"operationId": "7af420dc-f8b7-450f-8f6f-18b039aa3cde",
"responses": {
"200": {
"description": "Successful call with status 200",
"content": {
"": {
"x-sample-entry": "000000000000000000000016"
}
},
"x-sample-entry": "000000000000000000000016"
}
},
"x-counters-per-source": {
"": {
"entries": 1,
"failures": 0,
"firstSeen": 1567750582.000002,
"lastSeen": 1567750582.000002,
"sumRT": 0.001,
"sumDuration": 0
}
},
"x-counters-total": {
"entries": 1,
"failures": 0,
"firstSeen": 1567750582.000002,
"lastSeen": 1567750582.000002,
"sumRT": 0.001,
"sumDuration": 0
},
"x-last-seen-ts": 1567750582.000002,
"x-sample-entry": "000000000000000000000016"
},
"parameters": [
{
"name": "parampatternId",
"in": "path",
"required": true,
"style": "simple",
"schema": {
"type": "string",
"pattern": "^prefix-gibberish-.+"
},
"examples": {
"example #0": {
"value": "prefix-gibberish-sfdlasdfkadf87sd93284q24r"
},
"example #1": {
"value": "prefix-gibberish-adslkfasdf89sa7dfasddafa8a98sd7kansdf"
},
"example #2": {
"value": "prefix-gibberish-4jk5l2345h2452l4352435jlk45"
},
"example #3": {
"value": "prefix-gibberish-84395h2j4k35hj243j5h2kl34h54k"
},
"example #4": {
"value": "prefix-gibberish-afterwards"
}
},
"x-sample-entry": "000000000000000000000019"
}
]
},
"/param-patterns/{parampatternId}/{param1}": {
"get": {
"tags": [
"param-patterns"
],
"summary": "/param-patterns/{parampatternId}/{param1}",
"description": "Kubeshark observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.001 seconds",
"operationId": "02a1771d-2d50-4a8c-8be2-29c7e59b8435",
"responses": {
"200": {
"description": "Successful call with status 200",
"content": {
"": {
"x-sample-entry": "000000000000000000000019"
}
},
"x-sample-entry": "000000000000000000000019"
}
},
"x-counters-per-source": {
"": {
"entries": 1,
"failures": 0,
"firstSeen": 1567750582.000002,
"lastSeen": 1567750582.000002,
"sumRT": 0.001,
"sumDuration": 0
}
},
"x-counters-total": {
"entries": 1,
"failures": 0,
"firstSeen": 1567750582.000002,
"lastSeen": 1567750582.000002,
"sumRT": 0.001,
"sumDuration": 0
},
"x-last-seen-ts": 1567750582.000002,
"x-sample-entry": "000000000000000000000019"
},
"parameters": [
{
"name": "param1",
"in": "path",
"required": true,
"style": "simple",
"schema": {
"type": "string"
},
"examples": {
"example #0": {
"value": "23421"
}
},
"x-sample-entry": "000000000000000000000019"
},
{
"name": "parampatternId",
"in": "path",
"required": true,
"style": "simple",
"schema": {
"type": "string",
"pattern": "^prefix-gibberish-.+"
},
"examples": {
"example #0": {
"value": "prefix-gibberish-sfdlasdfkadf87sd93284q24r"
},
"example #1": {
"value": "prefix-gibberish-adslkfasdf89sa7dfasddafa8a98sd7kansdf"
},
"example #2": {
"value": "prefix-gibberish-4jk5l2345h2452l4352435jlk45"
},
"example #3": {
"value": "prefix-gibberish-84395h2j4k35hj243j5h2kl34h54k"
},
"example #4": {
"value": "prefix-gibberish-afterwards"
}
},
"x-sample-entry": "000000000000000000000019"
}
]
},
"/{Id}": {
"get": {
"summary": "/{Id}",
"description": "Kubeshark observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.630 seconds",
"operationId": "77ec4910-d47a-46a5-8234-fb80a11034b4",
"responses": {
"200": {
"description": "Successful call with status 200",
"content": {
"application/json": {
"example": null,
"x-sample-entry": "000000000000000000000003"
}
},
"x-sample-entry": "000000000000000000000003"
}
},
"x-counters-per-source": {
"": {
"entries": 1,
"failures": 0,
"firstSeen": 1567750579.7471218,
"lastSeen": 1567750579.7471218,
"sumRT": 0.63,
"sumDuration": 0
}
},
"x-counters-total": {
"entries": 1,
"failures": 0,
"firstSeen": 1567750579.7471218,
"lastSeen": 1567750579.7471218,
"sumRT": 0.63,
"sumDuration": 0
},
"x-last-seen-ts": 1567750579.7471218,
"x-sample-entry": "000000000000000000000003"
},
"parameters": [
{
"name": "Id",
"in": "path",
"required": true,
"style": "simple",
"schema": {
"type": "string"
},
"examples": {
"example #0": {
"value": "e21f7112-3d3b-4632-9da3-a4af2e0e9166"
},
"example #1": {
"value": "952bea17-3776-11ea-9341-42010a84012a"
}
},
"x-sample-entry": "000000000000000000000003"
}
]
},
"/{Id}/sub1": {
"get": {
"summary": "/{Id}/sub1",
"description": "Kubeshark observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.111 seconds",
"operationId": "198675eb-9faf-407b-83fa-0483a730bbbe",
"responses": {
"200": {
"description": "Successful call with status 200",
"content": {
"text/html": {
"x-sample-entry": "000000000000000000000001"
}
},
"x-sample-entry": "000000000000000000000001"
}
},
"x-counters-per-source": {
"": {
"entries": 1,
"failures": 0,
"firstSeen": 1567750483.864529,
"lastSeen": 1567750483.864529,
"sumRT": 0.111,
"sumDuration": 0
}
},
"x-counters-total": {
"entries": 1,
"failures": 0,
"firstSeen": 1567750483.864529,
"lastSeen": 1567750483.864529,
"sumRT": 0.111,
"sumDuration": 0
},
"x-last-seen-ts": 1567750483.864529,
"x-sample-entry": "000000000000000000000001"
},
"parameters": [
{
"name": "Id",
"in": "path",
"required": true,
"style": "simple",
"schema": {
"type": "string"
},
"examples": {
"example #0": {
"value": "e21f7112-3d3b-4632-9da3-a4af2e0e9166"
},
"example #1": {
"value": "952bea17-3776-11ea-9341-42010a84012a"
}
},
"x-sample-entry": "000000000000000000000003"
}
]
},
"/{Id}/sub2": {
"get": {
"summary": "/{Id}/sub2",
"description": "Kubeshark observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.630 seconds",
"operationId": "31d880f1-152f-4dd6-84a7-463e13b694a5",
"responses": {
"200": {
"description": "Successful call with status 200",
"content": {
"application/json": {
"example": null,
"x-sample-entry": "000000000000000000000002"
}
},
"x-sample-entry": "000000000000000000000002"
}
},
"x-counters-per-source": {
"": {
"entries": 1,
"failures": 0,
"firstSeen": 1567750578.7471218,
"lastSeen": 1567750578.7471218,
"sumRT": 0.63,
"sumDuration": 0
}
},
"x-counters-total": {
"entries": 1,
"failures": 0,
"firstSeen": 1567750578.7471218,
"lastSeen": 1567750578.7471218,
"sumRT": 0.63,
"sumDuration": 0
},
"x-last-seen-ts": 1567750578.7471218,
"x-sample-entry": "000000000000000000000002"
},
"parameters": [
{
"name": "Id",
"in": "path",
"required": true,
"style": "simple",
"schema": {
"type": "string"
},
"examples": {
"example #0": {
"value": "e21f7112-3d3b-4632-9da3-a4af2e0e9166"
},
"example #1": {
"value": "952bea17-3776-11ea-9341-42010a84012a"
}
},
"x-sample-entry": "000000000000000000000003"
}
]
}
},
"x-counters-per-source": {
"": {
"entries": 19,
"failures": 0,
"firstSeen": 1567750483.864529,
"lastSeen": 1567750582.7471218,
"sumRT": 3.273999999999999,
"sumDuration": 2.0100011825561523
}
},
"x-counters-total": {
"entries": 19,
"failures": 0,
"firstSeen": 1567750483.864529,
"lastSeen": 1567750582.7471218,
"sumRT": 3.273999999999999,
"sumDuration": 2.0100011825561523
}
}

1787
agent/pkg/oas/oas-samples/trcc-api-service.json
View File

File diff suppressed because it is too large Load Diff

142
agent/pkg/oas/oas_generator.go
View File

@ -1,142 +0,0 @@
package oas
import (
"encoding/json"
"net/url"
"sync"
"github.com/kubeshark/kubeshark/agent/pkg/har"
"github.com/kubeshark/kubeshark/logger"
"github.com/kubeshark/worker/api"
)
var (
syncOnce sync.Once
instance *defaultOasGenerator
)
type OasGeneratorSink interface {
HandleEntry(kubesharkEntry *api.Entry)
}
type OasGenerator interface {
Start()
Stop()
IsStarted() bool
GetServiceSpecs() *sync.Map
}
type defaultOasGenerator struct {
started bool
serviceSpecs *sync.Map
maxExampleLen int
}
func GetDefaultOasGeneratorInstance(maxExampleLen int) *defaultOasGenerator {
syncOnce.Do(func() {
instance = NewDefaultOasGenerator(maxExampleLen)
logger.Log.Debug("OAS Generator Initialized")
})
return instance
}
func (g *defaultOasGenerator) Start() {
g.started = true
}
func (g *defaultOasGenerator) Stop() {
if !g.started {
return
}
g.started = false
g.reset()
}
func (g *defaultOasGenerator) IsStarted() bool {
return g.started
}
func (g *defaultOasGenerator) HandleEntry(kubesharkEntry *api.Entry) {
if !g.started {
return
}
if kubesharkEntry.Protocol.Name == "http" {
dest := kubesharkEntry.Destination.Name
if dest == "" {
logger.Log.Debugf("OAS: Unresolved entry %d", kubesharkEntry.Id)
return
}
entry, err := har.NewEntry(kubesharkEntry.Request, kubesharkEntry.Response, kubesharkEntry.StartTime, kubesharkEntry.ElapsedTime)
if err != nil {
logger.Log.Warningf("Failed to turn KubesharkEntry %d into HAR Entry: %s", kubesharkEntry.Id, err)
return
}
entryWSource := &EntryWithSource{
Entry: *entry,
Source: kubesharkEntry.Source.Name,
Destination: dest,
Id: kubesharkEntry.Id,
}
g.handleHARWithSource(entryWSource)
} else {
logger.Log.Debugf("OAS: Unsupported protocol in entry %d: %s", kubesharkEntry.Id, kubesharkEntry.Protocol.Name)
}
}
func (g *defaultOasGenerator) handleHARWithSource(entryWSource *EntryWithSource) {
entry := entryWSource.Entry
gen := g.getGen(entryWSource.Destination, entry.Request.URL)
opId, err := gen.feedEntry(entryWSource)
if err != nil {
txt, suberr := json.Marshal(entry)
if suberr == nil {
logger.Log.Debugf("Problematic entry: %s", txt)
}
logger.Log.Warningf("Failed processing entry %d: %s", entryWSource.Id, err)
return
}
logger.Log.Debugf("Handled entry %s as opId: %s", entryWSource.Id, opId) // TODO: set opId back to entry?
}
func (g *defaultOasGenerator) getGen(dest string, urlStr string) *SpecGen {
u, err := url.Parse(urlStr)
if err != nil {
logger.Log.Errorf("Failed to parse entry URL: %v, err: %v", urlStr, err)
}
val, found := g.serviceSpecs.Load(dest)
var gen *SpecGen
if !found {
gen = NewGen(u.Scheme + "://" + dest)
gen.MaxExampleLen = g.maxExampleLen
g.serviceSpecs.Store(dest, gen)
} else {
gen = val.(*SpecGen)
}
return gen
}
func (g *defaultOasGenerator) reset() {
g.serviceSpecs = &sync.Map{}
}
func (g *defaultOasGenerator) GetServiceSpecs() *sync.Map {
return g.serviceSpecs
}
func NewDefaultOasGenerator(maxExampleLen int) *defaultOasGenerator {
return &defaultOasGenerator{
started: false,
serviceSpecs: &sync.Map{},
maxExampleLen: maxExampleLen,
}
}

45
agent/pkg/oas/oas_generator_test.go
View File

@ -1,45 +0,0 @@
package oas
import (
"encoding/json"
"github.com/kubeshark/kubeshark/agent/pkg/har"
"testing"
"time"
)
func TestOASGen(t *testing.T) {
gen := GetDefaultOasGeneratorInstance(-1)
e := new(har.Entry)
err := json.Unmarshal([]byte(`{"startedDateTime": "20000101","request": {"url": "https://host/path", "method": "GET"}, "response": {"status": 200}}`), e)
if err != nil {
panic(err)
}
ews := &EntryWithSource{
Destination: "some",
Entry: *e,
}
gen.Start()
gen.handleHARWithSource(ews)
g, ok := gen.serviceSpecs.Load("some")
if !ok {
panic("Failed")
}
sg := g.(*SpecGen)
spec, err := sg.GetSpec()
if err != nil {
panic(err)
}
specText, _ := json.Marshal(spec)
t.Log(string(specText))
if !gen.IsStarted() {
t.Errorf("Should be started")
}
time.Sleep(100 * time.Millisecond)
gen.Stop()
}

747
agent/pkg/oas/specgen.go
View File

@ -1,747 +0,0 @@
package oas
import (
"encoding/json"
"errors"
"io"
"mime"
"mime/multipart"
"net/textproto"
"net/url"
"sort"
"strconv"
"strings"
"sync"
"github.com/chanced/openapi"
"github.com/google/uuid"
"github.com/kubeshark/kubeshark/logger"
"github.com/nav-inc/datetime"
"github.com/kubeshark/kubeshark/agent/pkg/har"
"time"
)
const LastSeenTS = "x-last-seen-ts"
const CountersTotal = "x-counters-total"
const CountersPerSource = "x-counters-per-source"
const SampleId = "x-sample-entry"
type EntryWithSource struct {
Source string
Destination string
Entry har.Entry
Id string
}
type reqResp struct { // hello, generics in Go
Req *har.Request
Resp *har.Response
}
type SpecGen struct {
MaxExampleLen int // -1 unlimited, 0 and above sets limit
oas *openapi.OpenAPI
tree *Node
lock sync.Mutex
}
func NewGen(server string) *SpecGen {
spec := new(openapi.OpenAPI)
spec.Version = "3.1.0"
info := openapi.Info{Title: server}
info.Version = "1.0"
spec.Info = &info
spec.Paths = &openapi.Paths{Items: map[openapi.PathValue]*openapi.PathObj{}}
spec.Servers = make([]*openapi.Server, 0)
spec.Servers = append(spec.Servers, &openapi.Server{URL: server})
gen := SpecGen{
oas: spec,
tree: new(Node),
MaxExampleLen: -1,
}
return &gen
}
func (g *SpecGen) StartFromSpec(oas *openapi.OpenAPI) {
g.oas = oas
g.tree = new(Node)
for pathStr, pathObj := range oas.Paths.Items {
pathSplit := strings.Split(string(pathStr), "/")
g.tree.getOrSet(pathSplit, pathObj, "")
// clean "last entry timestamp" markers from the past
for _, pathAndOp := range g.tree.listOps() {
delete(pathAndOp.op.Extensions, LastSeenTS)
}
}
}
func (g *SpecGen) feedEntry(entryWithSource *EntryWithSource) (string, error) {
g.lock.Lock()
defer g.lock.Unlock()
opId, err := g.handlePathObj(entryWithSource)
if err != nil {
return "", err
}
// NOTE: opId can be empty for some failed entries
return opId, err
}
func (g *SpecGen) GetSpec() (*openapi.OpenAPI, error) {
g.lock.Lock()
defer g.lock.Unlock()
g.tree.compact()
counters := CounterMaps{counterTotal: Counter{}, counterMapTotal: CounterMap{}}
for _, pathAndOp := range g.tree.listOps() {
opObj := pathAndOp.op
if opObj.Summary == "" {
opObj.Summary = pathAndOp.path
}
err := counters.processOp(opObj)
if err != nil {
return nil, err
}
}
err := counters.processOas(g.oas)
if err != nil {
return nil, err
}
// put paths back from tree into OAS
g.oas.Paths = g.tree.listPaths()
suggestTags(g.oas)
g.oas.Info.Description = setCounterMsgIfOk(g.oas.Info.Description, &counters.counterTotal)
// to make a deep copy, no better idea than marshal+unmarshal
specText, err := json.MarshalIndent(g.oas, "", "\t")
if err != nil {
return nil, err
}
spec := new(openapi.OpenAPI)
err = json.Unmarshal(specText, spec)
if err != nil {
return nil, err
}
return spec, err
}
func suggestTags(oas *openapi.OpenAPI) {
paths := getPathsKeys(oas.Paths.Items)
sort.Strings(paths) // make it stable in case of multiple candidates
for len(paths) > 0 {
group := make([]string, 0)
group = append(group, paths[0])
paths = paths[1:]
pathsClone := append(paths[:0:0], paths...)
for _, path := range pathsClone {
if getSimilarPrefix([]string{group[0], path}) != "" {
group = append(group, path)
paths = deleteFromSlice(paths, path)
}
}
common := getSimilarPrefix(group)
if len(group) > 1 {
for _, path := range group {
pathObj := oas.Paths.Items[openapi.PathValue(path)]
for _, op := range getOps(pathObj) {
if op.Tags == nil {
op.Tags = make([]string, 0)
}
// only add tags if not present
if len(op.Tags) == 0 {
op.Tags = append(op.Tags, common)
}
}
}
}
}
}
func getPathsKeys(mymap map[openapi.PathValue]*openapi.PathObj) []string {
keys := make([]string, len(mymap))
i := 0
for k := range mymap {
keys[i] = string(k)
i++
}
return keys
}
func (g *SpecGen) handlePathObj(entryWithSource *EntryWithSource) (string, error) {
entry := entryWithSource.Entry
urlParsed, err := url.Parse(entry.Request.URL)
if err != nil {
return "", err
}
if isExtIgnored(urlParsed.Path) {
logger.Log.Debugf("Dropped traffic entry due to ignored extension: %s", urlParsed.Path)
return "", nil
}
if entry.Request.Method == "OPTIONS" {
logger.Log.Debugf("Dropped traffic entry due to its method: %s %s", entry.Request.Method, urlParsed.Path)
return "", nil
}
ctype := getRespCtype(&entry.Response)
if isCtypeIgnored(ctype) {
logger.Log.Debugf("Dropped traffic entry due to ignored response ctype: %s", ctype)
return "", nil
}
if entry.Response.Status < 100 {
logger.Log.Debugf("Dropped traffic entry due to status<100: %s", entry.StartedDateTime)
return "", nil
}
if entry.Response.Status == 301 || entry.Response.Status == 308 {
logger.Log.Debugf("Dropped traffic entry due to permanent redirect status: %s", entry.StartedDateTime)
return "", nil
}
if entry.Response.Status == 502 || entry.Response.Status == 503 || entry.Response.Status == 504 {
logger.Log.Debugf("Dropped traffic entry due to temporary server error: %s", entry.StartedDateTime)
return "", nil
}
var split []string
if urlParsed.RawPath != "" {
split = strings.Split(urlParsed.RawPath, "/")
} else {
split = strings.Split(urlParsed.Path, "/")
}
node := g.tree.getOrSet(split, new(openapi.PathObj), entryWithSource.Id)
opObj, err := handleOpObj(entryWithSource, node.pathObj, g.MaxExampleLen)
if opObj != nil {
return opObj.OperationID, err
}
return "", err
}
func handleOpObj(entryWithSource *EntryWithSource, pathObj *openapi.PathObj, limit int) (*openapi.Operation, error) {
entry := entryWithSource.Entry
isSuccess := 100 <= entry.Response.Status && entry.Response.Status < 400
opObj, wasMissing, err := getOpObj(pathObj, entry.Request.Method, isSuccess)
if err != nil {
return nil, err
}
if !isSuccess && wasMissing {
logger.Log.Debugf("Dropped traffic entry due to failed status and no known endpoint at: %s", entry.StartedDateTime)
return nil, nil
}
err = handleRequest(&entry.Request, opObj, isSuccess, entryWithSource.Id, limit)
if err != nil {
return nil, err
}
err = handleResponse(&entry.Response, opObj, isSuccess, entryWithSource.Id, limit)
if err != nil {
return nil, err
}
err = handleCounters(opObj, isSuccess, entryWithSource)
if err != nil {
return nil, err
}
setSampleID(&opObj.Extensions, entryWithSource.Id)
return opObj, nil
}
func handleCounters(opObj *openapi.Operation, success bool, entryWithSource *EntryWithSource) error {
// TODO: if performance around DecodeExtension+SetExtension is bad, store counters as separate maps
counter := Counter{}
counterMap := CounterMap{}
prevTs := 0.0
if opObj.Extensions == nil {
opObj.Extensions = openapi.Extensions{}
} else {
if _, ok := opObj.Extensions.Extension(CountersTotal); ok {
err := opObj.Extensions.DecodeExtension(CountersTotal, &counter)
if err != nil {
return err
}
}
if _, ok := opObj.Extensions.Extension(CountersPerSource); ok {
err := opObj.Extensions.DecodeExtension(CountersPerSource, &counterMap)
if err != nil {
return err
}
}
if _, ok := opObj.Extensions.Extension(LastSeenTS); ok {
err := opObj.Extensions.DecodeExtension(LastSeenTS, &prevTs)
if err != nil {
return err
}
}
}
var counterPerSource *Counter
if existing, ok := counterMap[entryWithSource.Source]; ok {
counterPerSource = existing
} else {
counterPerSource = new(Counter)
counterMap[entryWithSource.Source] = counterPerSource
}
started, err := datetime.Parse(entryWithSource.Entry.StartedDateTime, time.UTC)
if err != nil {
return err
}
ts := float64(started.UnixNano()) / float64(time.Millisecond) / 1000
rt := float64(entryWithSource.Entry.Time) / 1000
dur := 0.0
if prevTs != 0 && ts >= prevTs {
dur = ts - prevTs
}
counter.addEntry(ts, rt, success, dur)
counterPerSource.addEntry(ts, rt, success, dur)
err = opObj.Extensions.SetExtension(LastSeenTS, ts)
if err != nil {
return err
}
err = opObj.Extensions.SetExtension(CountersTotal, counter)
if err != nil {
return err
}
err = opObj.Extensions.SetExtension(CountersPerSource, counterMap)
if err != nil {
return err
}
return nil
}
func handleRequest(req *har.Request, opObj *openapi.Operation, isSuccess bool, sampleId string, limit int) error {
// TODO: we don't handle the situation when header/qstr param can be defined on pathObj level. Also the path param defined on opObj
urlParsed, err := url.Parse(req.URL)
if err != nil {
return err
}
qs := make([]har.NVP, 0)
for name, vals := range urlParsed.Query() {
for _, val := range vals {
qs = append(qs, har.NVP{Name: name, Value: val})
}
}
if len(qs) != len(req.QueryString) {
logger.Log.Warningf("QStr params in HAR do not match URL: %s", req.URL)
}
qstrGW := nvParams{
In: openapi.InQuery,
Pairs: qs,
IsIgnored: func(name string) bool { return false },
GeneralizeName: func(name string) string { return name },
}
handleNameVals(qstrGW, &opObj.Parameters, false, sampleId)
hdrGW := nvParams{
In: openapi.InHeader,
Pairs: req.Headers,
IsIgnored: isHeaderIgnored,
GeneralizeName: strings.ToLower,
}
handleNameVals(hdrGW, &opObj.Parameters, true, sampleId)
if isSuccess {
reqBody, err := getRequestBody(req, opObj)
if err != nil {
return err
}
if reqBody != nil {
setSampleID(&reqBody.Extensions, sampleId)
if req.PostData.Text == "" {
reqBody.Required = false
} else {
reqCtype, _ := getReqCtype(req)
reqMedia, err := fillContent(reqResp{Req: req}, reqBody.Content, reqCtype, sampleId, limit)
if err != nil {
return err
}
_ = reqMedia
}
}
}
return nil
}
func handleResponse(resp *har.Response, opObj *openapi.Operation, isSuccess bool, sampleId string, limit int) error {
// TODO: we don't support "default" response
respObj, err := getResponseObj(resp, opObj, isSuccess)
if err != nil {
return err
}
setSampleID(&respObj.Extensions, sampleId)
handleRespHeaders(resp.Headers, respObj, sampleId)
respCtype := getRespCtype(resp)
respContent := respObj.Content
respMedia, err := fillContent(reqResp{Resp: resp}, respContent, respCtype, sampleId, limit)
if err != nil {
return err
}
_ = respMedia
return nil
}
func handleRespHeaders(reqHeaders []har.Header, respObj *openapi.ResponseObj, sampleId string) {
visited := map[string]*openapi.HeaderObj{}
for _, pair := range reqHeaders {
if isHeaderIgnored(pair.Name) {
continue
}
nameGeneral := strings.ToLower(pair.Name)
initHeaders(respObj)
objHeaders := respObj.Headers
param := findHeaderByName(&respObj.Headers, pair.Name)
if param == nil {
param = createHeader(openapi.TypeString)
objHeaders[nameGeneral] = param
}
exmp := &param.Examples
err := fillParamExample(&exmp, pair.Value)
if err != nil {
logger.Log.Warningf("Failed to add example to a parameter: %s", err)
}
visited[nameGeneral] = param
setSampleID(&param.Extensions, sampleId)
}
// maintain "required" flag
if respObj.Headers != nil {
for name, param := range respObj.Headers {
paramObj, err := param.ResolveHeader(headerResolver)
if err != nil {
logger.Log.Warningf("Failed to resolve param: %s", err)
continue
}
_, ok := visited[strings.ToLower(name)]
if !ok {
flag := false
paramObj.Required = &flag
}
}
}
}
func fillContent(reqResp reqResp, respContent openapi.Content, ctype string, sampleId string, limit int) (*openapi.MediaType, error) {
content, found := respContent[ctype]
if !found {
respContent[ctype] = &openapi.MediaType{}
content = respContent[ctype]
}
setSampleID(&content.Extensions, sampleId)
var text string
var isBinary bool
if reqResp.Req != nil {
isBinary, _, text = reqResp.Req.PostData.B64Decoded()
} else {
isBinary, _, text = reqResp.Resp.Content.B64Decoded()
}
if !isBinary && text != "" {
var exampleMsg []byte
// try treating it as json
anyVal, isJSON := anyJSON(text)
if isJSON {
// re-marshal with forced indent
if msg, err := json.MarshalIndent(anyVal, "", "\t"); err != nil {
panic("Failed to re-marshal value, super-strange")
} else {
exampleMsg = msg
}
} else {
if msg, err := json.Marshal(text); err != nil {
return nil, err
} else {
exampleMsg = msg
}
}
if ctype == "application/x-www-form-urlencoded" && reqResp.Req != nil {
handleFormDataUrlencoded(text, content)
} else if strings.HasPrefix(ctype, "multipart/form-data") && reqResp.Req != nil {
_, params := getReqCtype(reqResp.Req)
handleFormDataMultipart(text, content, params)
}
if len(exampleMsg) > len(content.Example) && (limit < 0 || len(exampleMsg) <= limit) {
content.Example = exampleMsg
}
}
return respContent[ctype], nil
}
func handleFormDataUrlencoded(text string, content *openapi.MediaType) {
formData, err := url.ParseQuery(text)
if err != nil {
logger.Log.Warningf("Could not decode urlencoded: %s", err)
return
}
parts := make([]PartWithBody, 0)
for name, vals := range formData {
for _, val := range vals {
part := new(multipart.Part)
part.Header = textproto.MIMEHeader{}
part.Header.Add("Content-Disposition", "form-data; name=\""+name+"\";")
parts = append(parts, PartWithBody{part: part, body: []byte(val)})
}
}
handleFormData(content, parts)
}
func handleFormData(content *openapi.MediaType, parts []PartWithBody) {
hadSchema := true
if content.Schema == nil {
hadSchema = false // will use it for required flags
content.Schema = new(openapi.SchemaObj)
content.Schema.Type = openapi.Types{openapi.TypeObject}
content.Schema.Properties = openapi.Schemas{}
}
props := &content.Schema.Properties
seenNames := map[string]struct{}{} // set equivalent in Go, yikes
for _, pwb := range parts {
name := pwb.part.FormName()
seenNames[name] = struct{}{}
existing, found := (*props)[name]
if !found {
existing = new(openapi.SchemaObj)
existing.Type = openapi.Types{openapi.TypeString}
(*props)[name] = existing
ctype := pwb.part.Header.Get("content-type")
if ctype != "" {
if existing.Keywords == nil {
existing.Keywords = map[string]json.RawMessage{}
}
existing.Keywords["contentMediaType"], _ = json.Marshal(ctype)
}
}
addSchemaExample(existing, string(pwb.body))
}
// handle required flag
if content.Schema.Required == nil {
if !hadSchema {
content.Schema.Required = make([]string, 0)
for name := range seenNames {
content.Schema.Required = append(content.Schema.Required, name)
}
sort.Strings(content.Schema.Required)
} // else it's a known schema with no required fields
} else {
content.Schema.Required = intersectSliceWithMap(content.Schema.Required, seenNames)
sort.Strings(content.Schema.Required)
}
}
type PartWithBody struct {
part *multipart.Part
body []byte
}
func handleFormDataMultipart(text string, content *openapi.MediaType, ctypeParams map[string]string) {
boundary, ok := ctypeParams["boundary"]
if !ok {
logger.Log.Errorf("Multipart header has no boundary")
return
}
mpr := multipart.NewReader(strings.NewReader(text), boundary)
parts := make([]PartWithBody, 0)
for {
part, err := mpr.NextPart()
if err == io.EOF {
break
}
if err != nil {
logger.Log.Errorf("Cannot parse multipart body: %v", err)
break
}
defer part.Close()
body, err := io.ReadAll(part)
if err != nil {
logger.Log.Errorf("Error reading multipart Part %s: %v", part.Header, err)
}
parts = append(parts, PartWithBody{part: part, body: body})
}
handleFormData(content, parts)
}
func getRespCtype(resp *har.Response) string {
var ctype string
ctype = resp.Content.MimeType
for _, hdr := range resp.Headers {
if strings.ToLower(hdr.Name) == "content-type" {
ctype = hdr.Value
}
}
mediaType, _, err := mime.ParseMediaType(ctype)
if err != nil {
return ""
}
return mediaType
}
func getReqCtype(req *har.Request) (ctype string, params map[string]string) {
ctype = req.PostData.MimeType
for _, hdr := range req.Headers {
if strings.ToLower(hdr.Name) == "content-type" {
ctype = hdr.Value
}
}
if ctype == "" {
return "", map[string]string{}
}
mediaType, params, err := mime.ParseMediaType(ctype)
if err != nil {
logger.Log.Errorf("Cannot parse Content-Type header %q: %v", ctype, err)
return "", map[string]string{}
}
return mediaType, params
}
func getResponseObj(resp *har.Response, opObj *openapi.Operation, isSuccess bool) (*openapi.ResponseObj, error) {
statusStr := strconv.Itoa(resp.Status)
var response openapi.Response
response, found := opObj.Responses[statusStr]
if !found {
if opObj.Responses == nil {
opObj.Responses = map[string]openapi.Response{}
}
opObj.Responses[statusStr] = &openapi.ResponseObj{Content: map[string]*openapi.MediaType{}}
response = opObj.Responses[statusStr]
}
resResponse, err := response.ResolveResponse(responseResolver)
if err != nil {
return nil, err
}
if isSuccess {
resResponse.Description = "Successful call with status " + statusStr
} else {
resResponse.Description = "Failed call with status " + statusStr
}
return resResponse, nil
}
func getRequestBody(req *har.Request, opObj *openapi.Operation) (*openapi.RequestBodyObj, error) {
if opObj.RequestBody == nil {
// create if there is body in request
if req.PostData.Text != "" {
opObj.RequestBody = &openapi.RequestBodyObj{Description: "Generic request body", Required: true, Content: map[string]*openapi.MediaType{}}
} else {
return nil, nil
}
}
reqBody, err := opObj.RequestBody.ResolveRequestBody(reqBodyResolver)
if err != nil {
return nil, err
}
return reqBody, nil
}
func getOpObj(pathObj *openapi.PathObj, method string, createIfNone bool) (*openapi.Operation, bool, error) {
method = strings.ToLower(method)
var op **openapi.Operation
switch method {
case "get":
op = &pathObj.Get
case "put":
op = &pathObj.Put
case "post":
op = &pathObj.Post
case "delete":
op = &pathObj.Delete
case "options":
op = &pathObj.Options
case "head":
op = &pathObj.Head
case "patch":
op = &pathObj.Patch
case "trace":
op = &pathObj.Trace
default:
return nil, false, errors.New("unsupported HTTP method: " + method)
}
isMissing := false
if *op == nil {
isMissing = true
if createIfNone {
*op = &openapi.Operation{Responses: map[string]openapi.Response{}}
newUUID := uuid.New().String()
(**op).OperationID = newUUID
} else {
return nil, isMissing, nil
}
}
return *op, isMissing, nil
}

267
agent/pkg/oas/specgen_test.go
View File

@ -1,267 +0,0 @@
package oas
import (
"encoding/json"
"io"
"os"
"regexp"
"strings"
"sync"
"testing"
"time"
"github.com/chanced/openapi"
"github.com/kubeshark/kubeshark/agent/pkg/har"
"github.com/kubeshark/kubeshark/logger"
"github.com/wI2L/jsondiff"
)
// if started via env, write file into subdir
func outputSpec(label string, spec *openapi.OpenAPI, t *testing.T) string {
content, err := json.MarshalIndent(spec, "", " ")
if err != nil {
panic(err)
}
if os.Getenv("KUBESHARK_OAS_WRITE_FILES") != "" {
path := "./oas-samples"
err := os.MkdirAll(path, 0o755)
if err != nil {
panic(err)
}
err = os.WriteFile(path+"/"+label+".json", content, 0644)
if err != nil {
panic(err)
}
t.Logf("Written: %s", label)
} else {
t.Logf("%s", string(content))
}
return string(content)
}
func TestEntries(t *testing.T) {
//logger.InitLoggerStd(logging.INFO) causes race condition
files, err := getFiles("./test_artifacts/")
if err != nil {
t.Log(err)
t.FailNow()
}
gen := NewDefaultOasGenerator(-1)
gen.serviceSpecs = new(sync.Map)
loadStartingOAS("test_artifacts/catalogue.json", "catalogue", gen.serviceSpecs)
loadStartingOAS("test_artifacts/trcc.json", "trcc-api-service", gen.serviceSpecs)
go func() {
for {
time.Sleep(1 * time.Second)
gen.serviceSpecs.Range(func(key, val interface{}) bool {
svc := key.(string)
t.Logf("Getting spec for %s", svc)
gen := val.(*SpecGen)
_, err := gen.GetSpec()
if err != nil {
t.Error(err)
}
return true
})
}
}()
cnt, err := feedEntries(files, true, gen)
if err != nil {
t.Log(err)
t.Fail()
}
svcs := strings.Builder{}
gen.serviceSpecs.Range(func(key, val interface{}) bool {
gen := val.(*SpecGen)
svc := key.(string)
svcs.WriteString(svc + ",")
spec, err := gen.GetSpec()
if err != nil {
t.Log(err)
t.FailNow()
return false
}
err = spec.Validate()
if err != nil {
specText, _ := json.MarshalIndent(spec, "", "\t")
t.Log(string(specText))
t.Log(err)
t.FailNow()
}
return true
})
gen.serviceSpecs.Range(func(key, val interface{}) bool {
svc := key.(string)
gen := val.(*SpecGen)
spec, err := gen.GetSpec()
if err != nil {
t.Log(err)
t.FailNow()
}
outputSpec(svc, spec, t)
err = spec.Validate()
if err != nil {
t.Log(err)
t.FailNow()
}
return true
})
logger.Log.Infof("Total entries: %d", cnt)
}
func TestFileSingle(t *testing.T) {
gen := NewDefaultOasGenerator(-1)
gen.serviceSpecs = new(sync.Map)
// loadStartingOAS()
file := "test_artifacts/params.har"
files := []string{file}
cnt, err := feedEntries(files, true, gen)
if err != nil {
logger.Log.Warning("Failed processing file: " + err.Error())
t.Fail()
}
gen.serviceSpecs.Range(func(key, val interface{}) bool {
svc := key.(string)
gen := val.(*SpecGen)
spec, err := gen.GetSpec()
if err != nil {
t.Log(err)
t.FailNow()
}
specText := outputSpec(svc, spec, t)
err = spec.Validate()
if err != nil {
t.Log(err)
t.FailNow()
}
expected, err := os.ReadFile(file + ".spec.json")
if err != nil {
t.Errorf(err.Error())
t.FailNow()
}
patFloatPrecision := regexp.MustCompile(`(\d+\.\d{1,2})(\d*)`)
expected = []byte(patUuid4.ReplaceAllString(string(expected), "<UUID4>"))
specText = patUuid4.ReplaceAllString(specText, "<UUID4>")
expected = []byte(patFloatPrecision.ReplaceAllString(string(expected), "$1"))
specText = patFloatPrecision.ReplaceAllString(specText, "$1")
diff, err := jsondiff.CompareJSON(expected, []byte(specText))
if err != nil {
t.Errorf(err.Error())
t.FailNow()
}
if os.Getenv("KUBESHARK_OAS_WRITE_FILES") != "" {
err = os.WriteFile(file+".spec.json", []byte(specText), 0644)
if err != nil {
panic(err)
}
}
if len(diff) > 0 {
t.Errorf("Generated spec does not match expected:\n%s", diff.String())
}
return true
})
logger.Log.Infof("Processed entries: %d", cnt)
}
func loadStartingOAS(file string, label string, specs *sync.Map) {
fd, err := os.Open(file)
if err != nil {
panic(err)
}
defer fd.Close()
data, err := io.ReadAll(fd)
if err != nil {
panic(err)
}
var doc *openapi.OpenAPI
err = json.Unmarshal(data, &doc)
if err != nil {
panic(err)
}
gen := NewGen(label)
gen.StartFromSpec(doc)
specs.Store(label, gen)
}
func TestEntriesNegative(t *testing.T) {
gen := NewDefaultOasGenerator(-1)
gen.serviceSpecs = new(sync.Map)
files := []string{"invalid"}
_, err := feedEntries(files, false, gen)
if err == nil {
t.Logf("Should have failed")
t.Fail()
}
}
func TestEntriesPositive(t *testing.T) {
gen := NewDefaultOasGenerator(-1)
gen.serviceSpecs = new(sync.Map)
files := []string{"test_artifacts/params.har"}
_, err := feedEntries(files, false, gen)
if err != nil {
t.Logf("Failed")
t.Fail()
}
}
func TestLoadValidHAR(t *testing.T) {
inp := `{"startedDateTime": "2021-02-03T07:48:12.959000+00:00", "time": 1, "request": {"method": "GET", "url": "http://unresolved_target/1.0.0/health", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [], "queryString": [], "headersSize": -1, "bodySize": -1}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [], "content": {"size": 2, "mimeType": "", "text": "OK"}, "redirectURL": "", "headersSize": -1, "bodySize": 2}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 1}}`
var entry *har.Entry
var err = json.Unmarshal([]byte(inp), &entry)
if err != nil {
t.Logf("Failed to decode entry: %s", err)
t.FailNow() // demonstrates the problem of `martian` HAR library
}
}
func TestLoadValid3_1(t *testing.T) {
fd, err := os.Open("test_artifacts/catalogue.json")
if err != nil {
t.Log(err)
t.FailNow()
}
defer fd.Close()
data, err := io.ReadAll(fd)
if err != nil {
t.Log(err)
t.FailNow()
}
var oas openapi.OpenAPI
err = json.Unmarshal(data, &oas)
if err != nil {
t.Log(err)
t.FailNow()
}
}

51
agent/pkg/oas/test_artifacts/catalogue.json
View File

@ -1,51 +0,0 @@
{
"openapi": "3.1.0",
"info": {
"title": "Preloaded",
"version": "0.1",
"description": "Test file for loading pre-existing OAS"
},
"paths": {
"/catalogue/{id}": {
"parameters": [
{
"name": "id",
"in": "path",
"required": true,
"style": "simple",
"schema": {
"type": "string"
},
"example": "some-uuid-maybe"
}
],
"get": {
"parameters": [ {
"name": "non-required-header",
"in": "header",
"required": true,
"style": "simple",
"schema": {
"type": "string"
},
"example": "some-uuid-maybe"
}
]
}
},
"/catalogue/{id}/details": {
"parameters": [
{
"name": "id",
"in": "path",
"style": "simple",
"required": true,
"schema": {
"type": "string"
},
"example": "some-uuid-maybe"
}
]
}
}
}

24969
agent/pkg/oas/test_artifacts/example.har
View File

File diff suppressed because one or more lines are too long

13
agent/pkg/oas/test_artifacts/example.ldjson
View File

@ -1,13 +0,0 @@
{"messageType": "http", "_source": "some-source", ",firstMessageTime": 1627298057.784151, "lastMessageTime": 1627298065.729303, "messageCount": 12}
{"_id": "", "startedDateTime": "2021-07-26T11:14:17.78415179Z", "time": 13, "request": {"method": "GET", "url": "http://catalogue/catalogue/size?tags=", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "x-some", "value": "demo val"},{"name": "Host", "value": "catalogue"}, {"name": "Connection", "value": "close"}], "queryString": [{"name": "tags", "value": ""}], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Content-Type", "value": "application/json; charset=utf-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:17 GMT"}, {"name": "Content-Length", "value": "22"}], "content": {"size": 22, "mimeType": "application/json; charset=utf-8", "text": "eyJlcnIiOm51bGwsInNpemUiOjl9", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": 22}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 13}}
{"_id": "", "startedDateTime": "2021-07-26T11:14:17.784918698Z", "time": 19, "request": {"method": "GET", "url": "http://catalogue/catalogue?page=1&size=6&tags=", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Connection", "value": "close"}, {"name": "Host", "value": "catalogue"}], "queryString": [{"name": "page", "value": "1"}, {"name": "size", "value": "6"}, {"name": "tags", "value": ""}], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Content-Type", "value": "application/json; charset=utf-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:17 GMT"}, {"name": "Content-Length", "value": "1927"}], "content": {"size": 1927, "mimeType": "application/json; charset=utf-8", "text": "W3siaWQiOiIwM2ZlZjZhYy0xODk2LTRjZTgtYmQ2OS1iNzk4Zjg1YzZlMGIiLCJuYW1lIjoiSG9seSIsImRlc2NyaXB0aW9uIjoiU29ja3MgZml0IGZvciBhIE1lc3NpYWguIFlvdSB0b28gY2FuIGV4cGVyaWVuY2Ugd2Fsa2luZyBpbiB3YXRlciB3aXRoIHRoZXNlIHNwZWNpYWwgZWRpdGlvbiBiZWF1dGllcy4gRWFjaCBob2xlIGlzIGxvdmluZ2x5IHByb2dnbGVkIHRvIGxlYXZlIHNtb290aCBlZGdlcy4gVGhlIG9ubHkgc29jayBhcHByb3ZlZCBieSBhIGhpZ2hlciBwb3dlci4iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9ob2x5XzEuanBlZyIsIi9jYXRhbG9ndWUvaW1hZ2VzL2hvbHlfMi5qcGVnIl0sInByaWNlIjo5OS45OSwiY291bnQiOjEsInRhZyI6WyJhY3Rpb24iLCJtYWdpYyJdfSx7ImlkIjoiMzM5NWE0M2UtMmQ4OC00MGRlLWI5NWYtZTAwZTE1MDIwODViIiwibmFtZSI6IkNvbG91cmZ1bCIsImRlc2NyaXB0aW9uIjoicHJvaWRlbnQgb2NjYWVjYXQgaXJ1cmUgZXQgZXhjZXB0ZXVyIGxhYm9yZSBtaW5pbSBuaXNpIGFtZXQgaXJ1cmUiLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9jb2xvdXJmdWxfc29ja3MuanBnIiwiL2NhdGFsb2d1ZS9pbWFnZXMvY29sb3VyZnVsX3NvY2tzLmpwZyJdLCJwcmljZSI6MTgsImNvdW50Ijo0MzgsInRhZyI6WyJicm93biIsImJsdWUiXX0seyJpZCI6IjUxMGEwZDdlLThlODMtNDE5My1iNDgzLWUyN2UwOWRkYzM0ZCIsIm5hbWUiOiJTdXBlclNwb3J0IFhMIiwiZGVzY3JpcHRpb24iOiJSZWFkeSBmb3IgYWN0aW9uLiBFbmdpbmVlcnM6IGJlIHJlYWR5IHRvIHNtYXNoIHRoYXQgbmV4dCBidWchIEJlIHJlYWR5LCB3aXRoIHRoZXNlIHN1cGVyLWFjdGlvbi1zcG9ydC1tYXN0ZXJwaWVjZXMuIFRoaXMgcGFydGljdWxhciBlbmdpbmVlciB3YXMgY2hhc2VkIGF3YXkgZnJvbSB0aGUgb2ZmaWNlIHdpdGggYSBzdGljay4iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9wdW1hXzEuanBlZyIsIi9jYXRhbG9ndWUvaW1hZ2VzL3B1bWFfMi5qcGVnIl0sInByaWNlIjoxNSwiY291bnQiOjgyMCwidGFnIjpbInNwb3J0IiwiZm9ybWFsIiwiYmxhY2siXX0seyJpZCI6IjgwOGEyZGUxLTFhYWEtNGMyNS1hOWI5LTY2MTJlOGYyOWEzOCIsIm5hbWUiOiJDcm9zc2VkIiwiZGVzY3JpcHRpb24iOiJBIG1hdHVyZSBzb2NrLCBjcm9zc2VkLCB3aXRoIGFuIGFpciBvZiBub25jaGFsYW5jZS4iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9jcm9zc18xLmpwZWciLCIvY2F0YWxvZ3VlL2ltYWdlcy9jcm9zc18yLmpwZWciXSwicHJpY2UiOjE3LjMyLCJjb3VudCI6NzM4LCJ0YWciOlsiYmx1ZSIsImFjdGlvbiIsInJlZCIsImZvcm1hbCJdfSx7ImlkIjoiODE5ZTFmYmYtOGI3ZS00ZjZkLTgxMWYtNjkzNTM0OTE2YThiIiwibmFtZSI6IkZpZ3Vlcm9hIiwiZGVzY3JpcHRpb24iOiJlbmltIG9mZmljaWEgYWxpcXVhIGV4Y2VwdGV1ciBlc3NlIGRlc2VydW50IHF1aXMgYWxpcXVpcCBub3N0cnVkIGFuaW0iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9XQVQuanBnIiwiL2NhdGFsb2d1ZS9pbWFnZXMvV0FUMi5qcGciXSwicHJpY2UiOjE0LCJjb3VudCI6ODA4LCJ0YWciOlsiZ3JlZW4iLCJmb3JtYWwiLCJibHVlIl19LHsiaWQiOiI4MzdhYjE0MS0zOTllLTRjMWYtOWFiYy1iYWNlNDAyOTZiYWMiLCJuYW1lIjoiQ2F0IHNvY2tzIiwiZGVzY3JpcHRpb24iOiJjb25zZXF1YXQgYW1ldCBjdXBpZGF0YXQgbWluaW0gbGFib3J1bSB0ZW1wb3IgZWxpdCBleCBjb25zZXF1YXQgaW4iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9jYXRzb2Nrcy5qcGciLCIvY2F0YWxvZ3VlL2ltYWdlcy9jYXRzb2NrczIuanBnIl0sInByaWNlIjoxNSwiY291bnQiOjE3NSwidGFnIjpbImJyb3duIiwiZm9ybWFsIiwiZ3JlZW4iXX1dCg==", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": 1927}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 19}}
{"_id": "", "startedDateTime": "2021-07-26T11:14:17.78418182Z", "time": 7, "request": {"method": "GET", "url": "http://catalogue/tags", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Connection", "value": "close"}, {"name": "Host", "value": "catalogue"}], "queryString": [], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Content-Type", "value": "application/json; charset=utf-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:17 GMT"}, {"name": "Content-Length", "value": "107"}], "content": {"size": 107, "mimeType": "application/json; charset=utf-8", "text": "eyJlcnIiOm51bGwsInRhZ3MiOlsiYnJvd24iLCJnZWVrIiwiZm9ybWFsIiwiYmx1ZSIsInNraW4iLCJyZWQiLCJhY3Rpb24iLCJzcG9ydCIsImJsYWNrIiwibWFnaWMiLCJncmVlbiJdfQ==", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": 107}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 7}}
{"_id": "", "startedDateTime": "2021-07-26T11:14:18.131501482Z", "time": 5, "request": {"method": "GET", "url": "http://catalogue/catalogue/3395a43e-2d88-40de-b95f-e00e1502085b", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Connection", "value": "close"}, {"name": "Host", "value": "catalogue"}, {"name": "x-some", "value": "demoval"}], ",queryString": [], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Content-Type", "value": "application/json; charset=utf-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:17 GMT"}, {"name": "Content-Length", "value": "286"}], "content": {"size": 286, "mimeType": "application/json; charset=utf-8", "text": "eyJjb3VudCI6NDM4LCJkZXNjcmlwdGlvbiI6InByb2lkZW50IG9jY2FlY2F0IGlydXJlIGV0IGV4Y2VwdGV1ciBsYWJvcmUgbWluaW0gbmlzaSBhbWV0IGlydXJlIiwiaWQiOiIzMzk1YTQzZS0yZDg4LTQwZGUtYjk1Zi1lMDBlMTUwMjA4NWIiLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9jb2xvdXJmdWxfc29ja3MuanBnIiwiL2NhdGFsb2d1ZS9pbWFnZXMvY29sb3VyZnVsX3NvY2tzLmpwZyJdLCJuYW1lIjoiQ29sb3VyZnVsIiwicHJpY2UiOjE4LCJ0YWciOlsiYnJvd24iLCJibHVlIl19", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": 286}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 5}}
{"_id": "", "startedDateTime": "2021-07-26T11:14:18.379836908Z", "time": 14, "request": {"method": "GET", "url": "http://carts/carts/mHK0P7zTktmV1zv57iWAvCTd43FFMHap/items", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Connection", "value": "close"}, {"name": "Host", "value": "carts"}], "queryString": [], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "X-Application-Context", "value": "carts:80"}, {"name": "Content-Type", "value": "application/json;charset=UTF-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:17 GMT"}, {"name": "Transfer-Encoding", "value": "chunked"}], "content": {"size": 113, "mimeType": "application/json;charset=UTF-8", "text": "W3siaWQiOiI2MGZlOThmYjg2YzBmYzAwMDg2OWE5MGMiLCJpdGVtSWQiOiIzMzk1YTQzZS0yZDg4LTQwZGUtYjk1Zi1lMDBlMTUwMjA4NWIiLCJxdWFudGl0eSI6MSwidW5pdFByaWNlIjoxOC4wfV0=", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": -1}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 14}}
{"_id": "", "startedDateTime": "2021-07-26T11:14:22.920540124Z", "time": 3, "request": {"method": "GET", "url": "http://catalogue/catalogue/808a2de1-1aaa-4c25-a9b9-6612e8f29a38", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Connection", "value": "close"}, {"name": "Host", "value": "catalogue"}], "queryString": [], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Content-Type", "value": "application/json; charset=utf-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:22 GMT"}, {"name": "Content-Length", "value": "275"}], "content": {"size": 275, "mimeType": "application/json; charset=utf-8", "text": "eyJjb3VudCI6NzM4LCJkZXNjcmlwdGlvbiI6IkEgbWF0dXJlIHNvY2ssIGNyb3NzZWQsIHdpdGggYW4gYWlyIG9mIG5vbmNoYWxhbmNlLiIsImlkIjoiODA4YTJkZTEtMWFhYS00YzI1LWE5YjktNjYxMmU4ZjI5YTM4IiwiaW1hZ2VVcmwiOlsiL2NhdGFsb2d1ZS9pbWFnZXMvY3Jvc3NfMS5qcGVnIiwiL2NhdGFsb2d1ZS9pbWFnZXMvY3Jvc3NfMi5qcGVnIl0sIm5hbWUiOiJDcm9zc2VkIiwicHJpY2UiOjE3LjMyLCJ0YWciOlsiYmx1ZSIsInJlZCIsImFjdGlvbiIsImZvcm1hbCJdfQ==", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": 275}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 3}}
{"_id": "", "startedDateTime": "2021-07-26T11:14:22.921609501Z", "time": 3, "request": {"method": "GET", "url": "http://catalogue/catalogue?sort=id&size=3&tags=blue", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Connection", "value": "close"}, {"name": "Host", "value": "catalogue"}], "queryString": [{"name": "size", "value": "3"}, {"name": "tags", "value": "blue"}, {"name": "sort", "value": "id"}], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Content-Length", "value": "789"}, {"name": "Content-Type", "value": "application/json; charset=utf-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:22 GMT"}], "content": {"size": 789, "mimeType": "application/json; charset=utf-8", "text": "W3siaWQiOiIzMzk1YTQzZS0yZDg4LTQwZGUtYjk1Zi1lMDBlMTUwMjA4NWIiLCJuYW1lIjoiQ29sb3VyZnVsIiwiZGVzY3JpcHRpb24iOiJwcm9pZGVudCBvY2NhZWNhdCBpcnVyZSBldCBleGNlcHRldXIgbGFib3JlIG1pbmltIG5pc2kgYW1ldCBpcnVyZSIsImltYWdlVXJsIjpbIi9jYXRhbG9ndWUvaW1hZ2VzL2NvbG91cmZ1bF9zb2Nrcy5qcGciLCIvY2F0YWxvZ3VlL2ltYWdlcy9jb2xvdXJmdWxfc29ja3MuanBnIl0sInByaWNlIjoxOCwiY291bnQiOjQzOCwidGFnIjpbImJsdWUiXX0seyJpZCI6IjgwOGEyZGUxLTFhYWEtNGMyNS1hOWI5LTY2MTJlOGYyOWEzOCIsIm5hbWUiOiJDcm9zc2VkIiwiZGVzY3JpcHRpb24iOiJBIG1hdHVyZSBzb2NrLCBjcm9zc2VkLCB3aXRoIGFuIGFpciBvZiBub25jaGFsYW5jZS4iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9jcm9zc18xLmpwZWciLCIvY2F0YWxvZ3VlL2ltYWdlcy9jcm9zc18yLmpwZWciXSwicHJpY2UiOjE3LjMyLCJjb3VudCI6NzM4LCJ0YWciOlsiYmx1ZSJdfSx7ImlkIjoiODE5ZTFmYmYtOGI3ZS00ZjZkLTgxMWYtNjkzNTM0OTE2YThiIiwibmFtZSI6IkZpZ3Vlcm9hIiwiZGVzY3JpcHRpb24iOiJlbmltIG9mZmljaWEgYWxpcXVhIGV4Y2VwdGV1ciBlc3NlIGRlc2VydW50IHF1aXMgYWxpcXVpcCBub3N0cnVkIGFuaW0iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9XQVQuanBnIiwiL2NhdGFsb2d1ZS9pbWFnZXMvV0FUMi5qcGciXSwicHJpY2UiOjE0LCJjb3VudCI6ODA4LCJ0YWciOlsiYmx1ZSJdfV0K", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": 789}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 3}}
{"_id": "", "startedDateTime": "2021-07-26T11:14:22.923197848Z", "time": 3, "request": {"method": "GET", "url": "http://catalogue/catalogue/3395a43e-2d88-40de-b95f-e00e1502085b", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Host", "value": "catalogue"}, {"name": "Connection", "value": "close"}], "queryString": [], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Content-Type", "value": "application/json; charset=utf-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:22 GMT"}, {"name": "Content-Length", "value": "286"}], "content": {"size": 286, "mimeType": "application/json; charset=utf-8", "text": "eyJjb3VudCI6NDM4LCJkZXNjcmlwdGlvbiI6InByb2lkZW50IG9jY2FlY2F0IGlydXJlIGV0IGV4Y2VwdGV1ciBsYWJvcmUgbWluaW0gbmlzaSBhbWV0IGlydXJlIiwiaWQiOiIzMzk1YTQzZS0yZDg4LTQwZGUtYjk1Zi1lMDBlMTUwMjA4NWIiLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9jb2xvdXJmdWxfc29ja3MuanBnIiwiL2NhdGFsb2d1ZS9pbWFnZXMvY29sb3VyZnVsX3NvY2tzLmpwZyJdLCJuYW1lIjoiQ29sb3VyZnVsIiwicHJpY2UiOjE4LCJ0YWciOlsiYnJvd24iLCJibHVlIl19", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": 286}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 3}}
{"_id": "", "startedDateTime": "2021-07-26T11:14:23.175549218Z", "time": 26, "request": {"method": "GET", "url": "http://carts/carts/mHK0P7zTktmV1zv57iWAvCTd43FFMHap/items", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Connection", "value": "close"}, {"name": "Host", "value": "carts"}], "queryString": [], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "X-Application-Context", "value": "carts:80"}, {"name": "Content-Type", "value": "application/json;charset=UTF-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:22 GMT"}, {"name": "Transfer-Encoding", "value": "chunked"}], "content": {"size": 113, "mimeType": "application/json;charset=UTF-8", "text": "W3siaWQiOiI2MGZlOThmYjg2YzBmYzAwMDg2OWE5MGMiLCJpdGVtSWQiOiIzMzk1YTQzZS0yZDg4LTQwZGUtYjk1Zi1lMDBlMTUwMjA4NWIiLCJxdWFudGl0eSI6MSwidW5pdFByaWNlIjoxOC4wfV0=", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": -1}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 26}}
{"_id": "", "startedDateTime": "2021-07-26T11:14:25.239777333Z", "time": 10, "request": {"method": "GET", "url": "http://carts/carts/mHK0P7zTktmV1zv57iWAvCTd43FFMHap/items", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Connection", "value": "close"}, {"name": "Host", "value": "carts"}], "queryString": [], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Content-Type", "value": "application/json;charset=UTF-8"}, {"name": "Transfer-Encoding", "value": "chunked"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:25 GMT"}, {"name": "X-Application-Context", "value": "carts:80"}], "content": {"size": 113, "mimeType": "application/json;charset=UTF-8", "text": "W3siaWQiOiI2MGZlOThmYjg2YzBmYzAwMDg2OWE5MGMiLCJpdGVtSWQiOiIzMzk1YTQzZS0yZDg4LTQwZGUtYjk1Zi1lMDBlMTUwMjA4NWIiLCJxdWFudGl0eSI6MSwidW5pdFByaWNlIjoxOC4wfV0=", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": -1}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 10}}
{"_id": "", "startedDateTime": "2021-07-26T11:14:25.725866772Z", "time": 3, "request": {"method": "GET", "url": "http://catalogue/catalogue?size=5", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Connection", "value": "close"}, {"name": "Host", "value": "catalogue"}], "queryString": [{"name": "size", "value": "5"}], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Content-Length", "value": "1643"}, {"name": "Content-Type", "value": "application/json; charset=utf-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:25 GMT"}], "content": {"size": 1643, "mimeType": "application/json; charset=utf-8", "text": "W3siaWQiOiIwM2ZlZjZhYy0xODk2LTRjZTgtYmQ2OS1iNzk4Zjg1YzZlMGIiLCJuYW1lIjoiSG9seSIsImRlc2NyaXB0aW9uIjoiU29ja3MgZml0IGZvciBhIE1lc3NpYWguIFlvdSB0b28gY2FuIGV4cGVyaWVuY2Ugd2Fsa2luZyBpbiB3YXRlciB3aXRoIHRoZXNlIHNwZWNpYWwgZWRpdGlvbiBiZWF1dGllcy4gRWFjaCBob2xlIGlzIGxvdmluZ2x5IHByb2dnbGVkIHRvIGxlYXZlIHNtb290aCBlZGdlcy4gVGhlIG9ubHkgc29jayBhcHByb3ZlZCBieSBhIGhpZ2hlciBwb3dlci4iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9ob2x5XzEuanBlZyIsIi9jYXRhbG9ndWUvaW1hZ2VzL2hvbHlfMi5qcGVnIl0sInByaWNlIjo5OS45OSwiY291bnQiOjEsInRhZyI6WyJhY3Rpb24iLCJtYWdpYyJdfSx7ImlkIjoiMzM5NWE0M2UtMmQ4OC00MGRlLWI5NWYtZTAwZTE1MDIwODViIiwibmFtZSI6IkNvbG91cmZ1bCIsImRlc2NyaXB0aW9uIjoicHJvaWRlbnQgb2NjYWVjYXQgaXJ1cmUgZXQgZXhjZXB0ZXVyIGxhYm9yZSBtaW5pbSBuaXNpIGFtZXQgaXJ1cmUiLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9jb2xvdXJmdWxfc29ja3MuanBnIiwiL2NhdGFsb2d1ZS9pbWFnZXMvY29sb3VyZnVsX3NvY2tzLmpwZyJdLCJwcmljZSI6MTgsImNvdW50Ijo0MzgsInRhZyI6WyJicm93biIsImJsdWUiXX0seyJpZCI6IjUxMGEwZDdlLThlODMtNDE5My1iNDgzLWUyN2UwOWRkYzM0ZCIsIm5hbWUiOiJTdXBlclNwb3J0IFhMIiwiZGVzY3JpcHRpb24iOiJSZWFkeSBmb3IgYWN0aW9uLiBFbmdpbmVlcnM6IGJlIHJlYWR5IHRvIHNtYXNoIHRoYXQgbmV4dCBidWchIEJlIHJlYWR5LCB3aXRoIHRoZXNlIHN1cGVyLWFjdGlvbi1zcG9ydC1tYXN0ZXJwaWVjZXMuIFRoaXMgcGFydGljdWxhciBlbmdpbmVlciB3YXMgY2hhc2VkIGF3YXkgZnJvbSB0aGUgb2ZmaWNlIHdpdGggYSBzdGljay4iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9wdW1hXzEuanBlZyIsIi9jYXRhbG9ndWUvaW1hZ2VzL3B1bWFfMi5qcGVnIl0sInByaWNlIjoxNSwiY291bnQiOjgyMCwidGFnIjpbInNwb3J0IiwiZm9ybWFsIiwiYmxhY2siXX0seyJpZCI6IjgwOGEyZGUxLTFhYWEtNGMyNS1hOWI5LTY2MTJlOGYyOWEzOCIsIm5hbWUiOiJDcm9zc2VkIiwiZGVzY3JpcHRpb24iOiJBIG1hdHVyZSBzb2NrLCBjcm9zc2VkLCB3aXRoIGFuIGFpciBvZiBub25jaGFsYW5jZS4iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9jcm9zc18xLmpwZWciLCIvY2F0YWxvZ3VlL2ltYWdlcy9jcm9zc18yLmpwZWciXSwicHJpY2UiOjE3LjMyLCJjb3VudCI6NzM4LCJ0YWciOlsiYmx1ZSIsImFjdGlvbiIsInJlZCIsImZvcm1hbCJdfSx7ImlkIjoiODE5ZTFmYmYtOGI3ZS00ZjZkLTgxMWYtNjkzNTM0OTE2YThiIiwibmFtZSI6IkZpZ3Vlcm9hIiwiZGVzY3JpcHRpb24iOiJlbmltIG9mZmljaWEgYWxpcXVhIGV4Y2VwdGV1ciBlc3NlIGRlc2VydW50IHF1aXMgYWxpcXVpcCBub3N0cnVkIGFuaW0iLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9XQVQuanBnIiwiL2NhdGFsb2d1ZS9pbWFnZXMvV0FUMi5qcGciXSwicHJpY2UiOjE0LCJjb3VudCI6ODA4LCJ0YWciOlsiZ3JlZW4iLCJmb3JtYWwiLCJibHVlIl19XQo=", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": 1643}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 3}}
{"_id": "", "startedDateTime": "2021-07-26T11:14:25.729303217Z", "time": 3, "request": {"method": "GET", "url": "http://catalogue/catalogue/3395a43e-2d88-40de-b95f-e00e1502085b", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Connection", "value": "close"}, {"name": "Host", "value": "catalogue"}], "queryString": [], "headersSize": -1, "bodySize": 0}, "response": {"status": 200, "statusText": "OK", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [{"name": "Content-Type", "value": "application/json; charset=utf-8"}, {"name": "Date", "value": "Mon, 26 Jul 2021 11:14:25 GMT"}, {"name": "Content-Length", "value": "286"}], "content": {"size": 286, "mimeType": "application/json; charset=utf-8", "text": "eyJjb3VudCI6NDM4LCJkZXNjcmlwdGlvbiI6InByb2lkZW50IG9jY2FlY2F0IGlydXJlIGV0IGV4Y2VwdGV1ciBsYWJvcmUgbWluaW0gbmlzaSBhbWV0IGlydXJlIiwiaWQiOiIzMzk1YTQzZS0yZDg4LTQwZGUtYjk1Zi1lMDBlMTUwMjA4NWIiLCJpbWFnZVVybCI6WyIvY2F0YWxvZ3VlL2ltYWdlcy9jb2xvdXJmdWxfc29ja3MuanBnIiwiL2NhdGFsb2d1ZS9pbWFnZXMvY29sb3VyZnVsX3NvY2tzLmpwZyJdLCJuYW1lIjoiQ29sb3VyZnVsIiwicHJpY2UiOjE4LCJ0YWciOlsiYnJvd24iLCJibHVlIl19", "encoding": "base64"}, "redirectURL": "", "headersSize": -1, "bodySize": 286}, "cache": {}, "timings": {"send": -1, "wait": -1, "receive": 3}}

790
agent/pkg/oas/test_artifacts/params.har
View File

@ -1,790 +0,0 @@
{
"log": {
"version": "1.2",
"creator": {
"name": "mitmproxy har_dump",
"version": "0.1",
"comment": "mitmproxy version mitmproxy 4.0.4"
},
"entries": [
{
"startedDateTime": "2019-09-06T06:14:43.864529+00:00",
"time": 111,
"request": {
"method": "GET",
"url": "https://httpbin.org/e21f7112-3d3b-4632-9da3-a4af2e0e9166/sub1",
"httpVersion": "HTTP/1.1",
"cookies": [],
"headers": [],
"headersSize": 1542,
"bodySize": 0,
"queryString": []
},
"response": {
"status": 200,
"statusText": "OK",
"httpVersion": "HTTP/1.1",
"cookies": [],
"headers": [
],
"content": {
"mimeType": "text/html",
"text": "",
"size": 0
},
"redirectURL": "",
"headersSize": 245,
"bodySize": 39
},
"cache": {},
"timings": {
"send": 22,
"receive": 2,
"wait": 87,
"connect": -1,
"ssl": -1
},
"serverIPAddress": "54.210.29.33"
},
{
"startedDateTime": "2019-09-06T06:16:18.747122+00:00",
"time": 630,
"request": {
"method": "GET",
"url": "https://httpbin.org/952bea17-3776-11ea-9341-42010a84012a/sub2",
"httpVersion": "HTTP/1.1",
"cookies": [],
"headers": [],
"queryString": [],
"headersSize": 1542,
"bodySize": 0
},
"response": {
"status": 200,
"statusText": "OK",
"httpVersion": "HTTP/1.1",
"cookies": [],
"headers": [],
"content": {
"size": 39,
"compression": -20,
"mimeType": "application/json",
"text": "null"
},
"redirectURL": "",
"headersSize": 248,
"bodySize": 39
},
"cache": {},
"timings": {
"send": 14,
"receive": 4,
"wait": 350,
"connect": 262,
"ssl": -1
}
},
{
"startedDateTime": "2019-09-06T06:16:19.747122+00:00",
"time": 630,
"request": {
"method": "GET",
"url": "https://httpbin.org/952bea17-3776-11ea-9341-42010a84012a;mparam=matrixparam",
"httpVersion": "HTTP/1.1",
"cookies": [],
"headers": [],
"queryString": [],
"headersSize": 1542,
"bodySize": 0
},
"response": {
"status": 200,
"statusText": "OK",
"httpVersion": "HTTP/1.1",
"cookies": [],
"headers": [],
"content": {
"size": 39,
"compression": -20,
"mimeType": "application/json",
"text": "null"
},
"redirectURL": "",
"headersSize": 248,
"bodySize": 39
},
"cache": {},
"timings": {
"send": 14,
"receive": 4,
"wait": 350,
"connect": 262,
"ssl": -1
}
},
{
"startedDateTime": "2019-09-06T06:16:20.047122+00:00",
"time": 630,
"request": {
"method": "GET",
"url": "https://httpbin.org/appears-once",
"httpVersion": "HTTP/1.1",
"cookies": [],
"headers": [],
"queryString": [],
"headersSize": 1542,
"bodySize": 0
},
"response": {
"status": 200,
"statusText": "OK",
"httpVersion": "HTTP/1.1",
"cookies": [],
"headers": [],
"content": {
"size": 39,
"compression": -20,
"mimeType": "application/json",
"text": "null"
},
"redirectURL": "",
"headersSize": 248,
"bodySize": 39
},
"cache": {},
"timings": {
"send": 14,
"receive": 4,
"wait": 350,
"connect": 262,
"ssl": -1
}
},
{
"startedDateTime": "2019-09-06T06:16:20.747122+00:00",
"time": 630,
"request": {
"method": "GET",
"url": "https://httpbin.org/appears-twice",
"httpVersion": "HTTP/1.1",
"cookies": [],
"headers": [],
"queryString": [],
"headersSize": 1542,
"bodySize": 0
},
"response": {
"status": 200,
"statusText": "OK",
"httpVersion": "HTTP/1.1",
"cookies": [],
"headers": [],
"content": {
"size": 39,
"compression": -20,
"mimeType": "application/json",
"text": "null"
},
"redirectURL": "",
"headersSize": 248,
"bodySize": 39
},
"cache": {},
"timings": {
"send": 14,
"receive": 4,
"wait": 350,
"connect": 262,
"ssl": -1
}
},
{
"startedDateTime": "2019-09-06T06:16:21.747122+00:00",
"time": 630,
"request": {
"method": "GET",
"url": "https://httpbin.org/appears-twice",
"httpVersion": "HTTP/1.1",
"cookies": [],
"headers": [],
"queryString": [],
"headersSize": 1542,
"bodySize": 0
},
"response": {
"status": 200,
"statusText": "OK",
"httpVersion": "HTTP/1.1",
"cookies": [],
"headers": [],
"content": {
"size": 39,
"compression": -20,
"mimeType": "application/json",
"text": "null"
},
"redirectURL": "",
"headersSize": 248,
"bodySize": 39
},
"cache": {},
"timings": {
"send": 14,
"receive": 4,
"wait": 350,
"connect": 262,
"ssl": -1
}
},
{
"startedDateTime": "2019-09-06T06:16:20.747122+00:00",
"time": 1,
"request": {
"method": "POST",
"url": "https://httpbin.org/form-urlencoded",
"httpVersion": "",
"cookies": [],
"headers": [
{
"name": "Content-Type",
"value": "application/x-www-form-urlencoded"
}
],
"queryString": [],
"headersSize": -1,
"bodySize": -1,
"postData": {
"mimeType": "",
"text": "agent-id=ade&callback-url=&token=sometoken"
}
},
"response": {
"status": 200,
"statusText": "OK",
"httpVersion": "",
"cookies": [],
"headers": [
],
"content": {
"size": 0,
"mimeType": "",
"text": ""
},
"redirectURL": "",
"headersSize": -1,
"bodySize": 0
},
"cache": {},
"timings": {
"send": -1,
"wait": -1,
"receive": 1
}
},
{
"startedDateTime": "2019-09-06T06:16:21.747122+00:00",
"time": 1,
"request": {
"method": "POST",
"url": "https://httpbin.org/form-urlencoded",
"httpVersion": "",
"cookies": [],
"headers": [
{
"name": "Content-Type",
"value": "application/x-www-form-urlencoded"
}
],
"queryString": [],
"headersSize": -1,
"bodySize": -1,
"postData": {
"mimeType": "",
"text": "agent-id=ade&callback-url=&token=sometoken-second-val&optional=another"
}
},
"response": {
"status": 200,
"statusText": "OK",
"httpVersion": "",
"cookies": [],
"headers": [
],
"content": {
"size": 0,
"mimeType": "",
"text": ""
},
"redirectURL": "",
"headersSize": -1,
"bodySize": 0
},
"cache": {},
"timings": {
"send": -1,
"wait": -1,
"receive": 1
}
},
{
"startedDateTime": "2019-09-06T06:16:22.747122+00:00",
"time": 1,
"request": {
"method": "POST",
"url": "https://httpbin.org/form-multipart",
"httpVersion": "",
"cookies": [],
"headers": [
{
"name": "Content-Type",
"value": "multipart/form-data; boundary=BOUNDARY"
}
],
"queryString": [],
"headersSize": -1,
"bodySize": -1,
"postData": {
"mimeType": "",
"text": "--BOUNDARY\r\nContent-Disposition: form-data; name=\"file\"; filename=\"metadata.json\"\r\nContent-Type: application/json\r\n\r\n{\"functions\": 123}\r\n--BOUNDARY\r\nContent-Disposition: form-data; name=\"path\"\r\n\r\n/content/components\r\n--BOUNDARY--\r\n"
}
},
"response": {
"status": 200,
"statusText": "OK",
"httpVersion": "",
"cookies": [],
"headers": [
],
"content": {
"size": 62,
"mimeType": "",
"text": "{}"
},
"redirectURL": "",
"headersSize": -1,
"bodySize": 62
},
"cache": {},
"timings": {
"send": -1,
"wait": -1,
"receive": 1
}
},
{
"startedDateTime": "2019-09-06T06:16:21.757122+00:00",
"time": 1,
"request": {
"method": "POST",
"url": "https://httpbin.org/body-optional",
"httpVersion": "",
"cookies": [],
"headers": [
],
"queryString": [],
"headersSize": -1,
"bodySize": -1,
"postData": {
"mimeType": "",
"text": ""
}
},
"response": {
"status": 200,
"statusText": "OK",
"httpVersion": "",
"cookies": [],
"headers": [
],
"content": {
"size": 0,
"mimeType": "",
"text": ""
},
"redirectURL": "",
"headersSize": -1,
"bodySize": 0
},
"cache": {},
"timings": {
"send": -1,
"wait": -1,
"receive": 1
}
},
{
"startedDateTime": "2019-09-06T06:16:21.747122+00:00",
"time": 1,
"request": {
"method": "POST",
"url": "https://httpbin.org/body-optional",
"httpVersion": "",
"cookies": [],
"headers": [
{
"name": "Content-Type",
"value": "application/json"
}
],
"queryString": [],
"headersSize": -1,
"bodySize": -1,
"postData": {
"mimeType": "",
"text": "{\"key\", \"val\"}"
}
},
"response": {
"status": 200,
"statusText": "OK",
"httpVersion": "",
"cookies": [],
"headers": [
],
"content": {
"size": 0,
"mimeType": "",
"text": ""
},
"redirectURL": "",
"headersSize": -1,
"bodySize": 0
},
"cache": {},
"timings": {
"send": -1,
"wait": -1,
"receive": 1
}
},
{
"startedDateTime": "2019-09-06T06:16:21.757122+00:00",
"time": 1,
"request": {
"method": "POST",
"url": "https://httpbin.org/body-optional",
"httpVersion": "",
"cookies": [],
"headers": [
],
"queryString": [],
"headersSize": -1,
"bodySize": -1,
"postData": {
"mimeType": "",
"text": ""
}
},
"response": {
"status": 200,
"statusText": "OK",
"httpVersion": "",
"cookies": [],
"headers": [
],
"content": {
"size": 0,
"mimeType": "",
"text": ""
},
"redirectURL": "",
"headersSize": -1,
"bodySize": 0
},
"cache": {},
"timings": {
"send": -1,
"wait": -1,
"receive": 1
}
},
{
"startedDateTime": "2019-09-06T06:16:21.757122+00:00",
"time": 1,
"request": {
"method": "POST",
"url": "https://httpbin.org/body-required",
"httpVersion": "",
"cookies": [],
"headers": [
],
"queryString": [],
"headersSize": -1,
"bodySize": -1,
"postData": {
"mimeType": "",
"text": "body exists"
}
},
"response": {
"status": 200,
"statusText": "OK",
"httpVersion": "",
"cookies": [],
"headers": [
],
"content": {
"size": 0,
"mimeType": "",
"text": ""
},
"redirectURL": "",
"headersSize": -1,
"bodySize": 0
},
"cache": {},
"timings": {
"send": -1,
"wait": -1,
"receive": 1
}
},
{
"startedDateTime": "2019-09-06T06:16:22.000000+00:00",
"time": 1,
"request": {
"method": "GET",
"url": "https://httpbin.org/param-patterns/prefix-gibberish-fine/234324",
"httpVersion": "",
"cookies": [],
"headers": [
],
"queryString": [],
"headersSize": -1,
"bodySize": -1,
"postData": {
"mimeType": "",
"text": ""
}
},
"response": {
"status": 200,
"statusText": "OK",
"httpVersion": "",
"cookies": [],
"headers": [
],
"content": {
"size": 0,
"mimeType": "",
"text": ""
},
"redirectURL": "",
"headersSize": -1,
"bodySize": 0
},
"cache": {},
"timings": {
"send": -1,
"wait": -1,
"receive": 1
}
},
{
"startedDateTime": "2019-09-06T06:16:22.000001+00:00",
"time": 1,
"request": {
"method": "GET",
"url": "https://httpbin.org/param-patterns/prefix-gibberish-sfdlasdfkadf87sd93284q24r/1",
"httpVersion": "",
"cookies": [],
"headers": [
],
"queryString": [],
"headersSize": -1,
"bodySize": -1,
"postData": {
"mimeType": "",
"text": ""
}
},
"response": {
"status": 200,
"statusText": "OK",
"httpVersion": "",
"cookies": [],
"headers": [
],
"content": {
"size": 0,
"mimeType": "",
"text": ""
},
"redirectURL": "",
"headersSize": -1,
"bodySize": 0
},
"cache": {},
"timings": {
"send": -1,
"wait": -1,
"receive": 1
}
},
{
"startedDateTime": "2019-09-06T06:16:22.000002+00:00",
"time": 1,
"request": {
"method": "GET",
"url": "https://httpbin.org/param-patterns/prefix-gibberish-adslkfasdf89sa7dfasddafa8a98sd7kansdf/static",
"httpVersion": "",
"cookies": [],
"headers": [
],
"queryString": [],
"headersSize": -1,
"bodySize": -1,
"postData": {
"mimeType": "",
"text": ""
}
},
"response": {
"status": 200,
"statusText": "OK",
"httpVersion": "",
"cookies": [],
"headers": [
],
"content": {
"size": 0,
"mimeType": "",
"text": ""
},
"redirectURL": "",
"headersSize": -1,
"bodySize": 0
},
"cache": {},
"timings": {
"send": -1,
"wait": -1,
"receive": 1
}
},
{
"startedDateTime": "2019-09-06T06:16:22.000003+00:00",
"time": 1,
"request": {
"method": "GET",
"url": "https://httpbin.org/param-patterns/prefix-gibberish-4jk5l2345h2452l4352435jlk45",
"httpVersion": "",
"cookies": [],
"headers": [
],
"queryString": [],
"headersSize": -1,
"bodySize": -1,
"postData": {
"mimeType": "",
"text": ""
}
},
"response": {
"status": 200,
"statusText": "OK",
"httpVersion": "",
"cookies": [],
"headers": [
],
"content": {
"size": 0,
"mimeType": "",
"text": ""
},
"redirectURL": "",
"headersSize": -1,
"bodySize": 0
},
"cache": {},
"timings": {
"send": -1,
"wait": -1,
"receive": 1
}
},
{
"startedDateTime": "2019-09-06T06:16:22.000004+00:00",
"time": 1,
"request": {
"method": "GET",
"url": "https://httpbin.org/param-patterns/prefix-gibberish-84395h2j4k35hj243j5h2kl34h54k",
"httpVersion": "",
"cookies": [],
"headers": [
],
"queryString": [],
"headersSize": -1,
"bodySize": -1,
"postData": {
"mimeType": "",
"text": ""
}
},
"response": {
"status": 200,
"statusText": "OK",
"httpVersion": "",
"cookies": [],
"headers": [
],
"content": {
"size": 0,
"mimeType": "",
"text": ""
},
"redirectURL": "",
"headersSize": -1,
"bodySize": 0
},
"cache": {},
"timings": {
"send": -1,
"wait": -1,
"receive": 1
}
},
{
"startedDateTime": "2019-09-06T06:16:22.000002+00:00",
"time": 1,
"request": {
"method": "GET",
"url": "https://httpbin.org/param-patterns/prefix-gibberish-afterwards/23421",
"httpVersion": "",
"cookies": [],
"headers": [
],
"queryString": [],
"headersSize": -1,
"bodySize": -1,
"postData": {
"mimeType": "",
"text": ""
}
},
"response": {
"status": 200,
"statusText": "OK",
"httpVersion": "",
"cookies": [],
"headers": [
],
"content": {
"size": 0,
"mimeType": "",
"text": ""
},
"redirectURL": "",
"headersSize": -1,
"bodySize": 0
},
"cache": {},
"timings": {
"send": -1,
"wait": -1,
"receive": 1
}
}
]
}
}

897
agent/pkg/oas/test_artifacts/params.har.spec.json
View File

@ -1,897 +0,0 @@
{
"openapi": "3.1.0",
"info": {
"title": "https://httpbin.org",
"description": "Kubeshark observed 19 entries (0 failed), at 0.10 hits/s, average response time is 0.17 seconds",
"version": "1.0"
},
"servers": [
{
"url": "https://httpbin.org"
}
],
"paths": {
"/appears-once": {
"get": {
"summary": "/appears-once",
"description": "Kubeshark observed 1 entries (0 failed), at 0.00 hits/s, average response time is 0.63 seconds",
"operationId": "<UUID4>",
"responses": {
"200": {
"description": "Successful call with status 200",
"content": {
"application/json": {
"example": null,
"x-sample-entry": "000000000000000000000004"
}
},
"x-sample-entry": "000000000000000000000004"
}
},
"x-counters-per-source": {
"": {
"entries": 1,
"failures": 0,
"firstSeen": 1567750580.04,
"lastSeen": 1567750580.04,
"sumRT": 0.63,
"sumDuration": 0
}
},
"x-counters-total": {
"entries": 1,
"failures": 0,
"firstSeen": 1567750580.04,
"lastSeen": 1567750580.04,
"sumRT": 0.63,
"sumDuration": 0
},
"x-last-seen-ts": 1567750580.04,
"x-sample-entry": "000000000000000000000004"
}
},
"/appears-twice": {
"get": {
"summary": "/appears-twice",
"description": "Kubeshark observed 2 entries (0 failed), at 0.50 hits/s, average response time is 0.63 seconds",
"operationId": "<UUID4>",
"responses": {
"200": {
"description": "Successful call with status 200",
"content": {
"application/json": {
"example": null,
"x-sample-entry": "000000000000000000000006"
}
},
"x-sample-entry": "000000000000000000000006"
}
},
"x-counters-per-source": {
"": {
"entries": 2,
"failures": 0,
"firstSeen": 1567750580.74,
"lastSeen": 1567750581.74,
"sumRT": 1.26,
"sumDuration": 1
}
},
"x-counters-total": {
"entries": 2,
"failures": 0,
"firstSeen": 1567750580.74,
"lastSeen": 1567750581.74,
"sumRT": 1.26,
"sumDuration": 1
},
"x-last-seen-ts": 1567750581.74,
"x-sample-entry": "000000000000000000000006"
}
},
"/body-optional": {
"post": {
"summary": "/body-optional",
"description": "Kubeshark observed 3 entries (0 failed), at 0.00 hits/s, average response time is 0.00 seconds",
"operationId": "<UUID4>",
"responses": {
"200": {
"description": "Successful call with status 200",
"content": {
"": {
"x-sample-entry": "000000000000000000000012"
}
},
"x-sample-entry": "000000000000000000000012"
}
},
"x-counters-per-source": {
"": {
"entries": 3,
"failures": 0,
"firstSeen": 1567750581.74,
"lastSeen": 1567750581.75,
"sumRT": 0.00,
"sumDuration": 0.01
}
},
"x-counters-total": {
"entries": 3,
"failures": 0,
"firstSeen": 1567750581.74,
"lastSeen": 1567750581.75,
"sumRT": 0.00,
"sumDuration": 0.01
},
"x-last-seen-ts": 1567750581.75,
"x-sample-entry": "000000000000000000000012",
"requestBody": {
"description": "Generic request body",
"content": {
"application/json": {
"example": "{\"key\", \"val\"}",
"x-sample-entry": "000000000000000000000011"
}
},
"x-sample-entry": "000000000000000000000012"
}
}
},
"/body-required": {
"post": {
"summary": "/body-required",
"description": "Kubeshark observed 1 entries (0 failed), at 0.00 hits/s, average response time is 0.00 seconds",
"operationId": "<UUID4>",
"responses": {
"200": {
"description": "Successful call with status 200",
"content": {
"": {
"x-sample-entry": "000000000000000000000013"
}
},
"x-sample-entry": "000000000000000000000013"
}
},
"x-counters-per-source": {
"": {
"entries": 1,
"failures": 0,
"firstSeen": 1567750581.75,
"lastSeen": 1567750581.75,
"sumRT": 0.00,
"sumDuration": 0
}
},
"x-counters-total": {
"entries": 1,
"failures": 0,
"firstSeen": 1567750581.75,
"lastSeen": 1567750581.75,
"sumRT": 0.00,
"sumDuration": 0
},
"x-last-seen-ts": 1567750581.75,
"x-sample-entry": "000000000000000000000013",
"requestBody": {
"description": "Generic request body",
"content": {
"": {
"example": "body exists",
"x-sample-entry": "000000000000000000000013"
}
},
"required": true,
"x-sample-entry": "000000000000000000000013"
}
}
},
"/form-multipart": {
"post": {
"summary": "/form-multipart",
"description": "Kubeshark observed 1 entries (0 failed), at 0.00 hits/s, average response time is 0.00 seconds",
"operationId": "<UUID4>",
"responses": {
"200": {
"description": "Successful call with status 200",
"content": {
"": {
"example": {},
"x-sample-entry": "000000000000000000000009"
}
},
"x-sample-entry": "000000000000000000000009"
}
},
"x-counters-per-source": {
"": {
"entries": 1,
"failures": 0,
"firstSeen": 1567750582.74,
"lastSeen": 1567750582.74,
"sumRT": 0.00,
"sumDuration": 0
}
},
"x-counters-total": {
"entries": 1,
"failures": 0,
"firstSeen": 1567750582.74,
"lastSeen": 1567750582.74,
"sumRT": 0.00,
"sumDuration": 0
},
"x-last-seen-ts": 1567750582.74,
"x-sample-entry": "000000000000000000000009",
"requestBody": {
"description": "Generic request body",
"content": {
"multipart/form-data": {
"schema": {
"type": "object",
"required": [
"file",
"path"
],
"properties": {
"file": {
"type": "string",
"contentMediaType": "application/json",
"examples": [
"{\"functions\": 123}"
]
},
"path": {
"type": "string",
"examples": [
"/content/components"
]
}
}
},
"example": "--BOUNDARY\r\nContent-Disposition: form-data; name=\"file\"; filename=\"metadata.json\"\r\nContent-Type: application/json\r\n\r\n{\"functions\": 123}\r\n--BOUNDARY\r\nContent-Disposition: form-data; name=\"path\"\r\n\r\n/content/components\r\n--BOUNDARY--\r\n",
"x-sample-entry": "000000000000000000000009"
}
},
"required": true,
"x-sample-entry": "000000000000000000000009"
}
}
},
"/form-urlencoded": {
"post": {
"summary": "/form-urlencoded",
"description": "Kubeshark observed 2 entries (0 failed), at 0.50 hits/s, average response time is 0.00 seconds",
"operationId": "<UUID4>",
"responses": {
"200": {
"description": "Successful call with status 200",
"content": {
"": {
"x-sample-entry": "000000000000000000000008"
}
},
"x-sample-entry": "000000000000000000000008"
}
},
"x-counters-per-source": {
"": {
"entries": 2,
"failures": 0,
"firstSeen": 1567750580.74,
"lastSeen": 1567750581.74,
"sumRT": 0.00,
"sumDuration": 1
}
},
"x-counters-total": {
"entries": 2,
"failures": 0,
"firstSeen": 1567750580.74,
"lastSeen": 1567750581.74,
"sumRT": 0.00,
"sumDuration": 1
},
"x-last-seen-ts": 1567750581.74,
"x-sample-entry": "000000000000000000000008",
"requestBody": {
"description": "Generic request body",
"content": {
"application/x-www-form-urlencoded": {
"schema": {
"type": "object",
"required": [
"agent-id",
"callback-url",
"token"
],
"properties": {
"agent-id": {
"type": "string",
"examples": [
"ade"
]
},
"callback-url": {
"type": "string",
"examples": [
""
]
},
"optional": {
"type": "string",
"examples": [
"another"
]
},
"token": {
"type": "string",
"examples": [
"sometoken",
"sometoken-second-val"
]
}
}
},
"example": "agent-id=ade\u0026callback-url=\u0026token=sometoken-second-val\u0026optional=another",
"x-sample-entry": "000000000000000000000008"
}
},
"required": true,
"x-sample-entry": "000000000000000000000008"
}
}
},
"/param-patterns/prefix-gibberish-fine/{prefixgibberishfineId}": {
"get": {
"tags": [
"param-patterns"
],
"summary": "/param-patterns/prefix-gibberish-fine/{prefixgibberishfineId}",
"description": "Kubeshark observed 1 entries (0 failed), at 0.00 hits/s, average response time is 0.00 seconds",
"operationId": "<UUID4>",
"responses": {
"200": {
"description": "Successful call with status 200",
"content": {
"": {
"x-sample-entry": "000000000000000000000014"
}
},
"x-sample-entry": "000000000000000000000014"
}
},
"x-counters-per-source": {
"": {
"entries": 1,
"failures": 0,
"firstSeen": 1567750582,
"lastSeen": 1567750582,
"sumRT": 0.00,
"sumDuration": 0
}
},
"x-counters-total": {
"entries": 1,
"failures": 0,
"firstSeen": 1567750582,
"lastSeen": 1567750582,
"sumRT": 0.00,
"sumDuration": 0
},
"x-last-seen-ts": 1567750582,
"x-sample-entry": "000000000000000000000014"
},
"parameters": [
{
"name": "prefixgibberishfineId",
"in": "path",
"required": true,
"style": "simple",
"schema": {
"type": "string"
},
"examples": {
"example #0": {
"value": "234324"
}
},
"x-sample-entry": "000000000000000000000014"
}
]
},
"/param-patterns/{parampatternId}": {
"get": {
"tags": [
"param-patterns"
],
"summary": "/param-patterns/{parampatternId}",
"description": "Kubeshark observed 2 entries (0 failed), at 0.00 hits/s, average response time is 0.00 seconds",
"operationId": "<UUID4>",
"responses": {
"200": {
"description": "Successful call with status 200",
"content": {
"": {
"x-sample-entry": "000000000000000000000018"
}
},
"x-sample-entry": "000000000000000000000018"
}
},
"x-counters-per-source": {
"": {
"entries": 2,
"failures": 0,
"firstSeen": 1567750582.00,
"lastSeen": 1567750582.00,
"sumRT": 0.00,
"sumDuration": 9.53e-7
}
},
"x-counters-total": {
"entries": 2,
"failures": 0,
"firstSeen": 1567750582.00,
"lastSeen": 1567750582.00,
"sumRT": 0.00,
"sumDuration": 9.53e-7
},
"x-last-seen-ts": 1567750582.00,
"x-sample-entry": "000000000000000000000018"
},
"parameters": [
{
"name": "parampatternId",
"in": "path",
"required": true,
"style": "simple",
"schema": {
"type": "string",
"pattern": "^prefix-gibberish-.+"
},
"examples": {
"example #0": {
"value": "prefix-gibberish-sfdlasdfkadf87sd93284q24r"
},
"example #1": {
"value": "prefix-gibberish-adslkfasdf89sa7dfasddafa8a98sd7kansdf"
},
"example #2": {
"value": "prefix-gibberish-4jk5l2345h2452l4352435jlk45"
},
"example #3": {
"value": "prefix-gibberish-84395h2j4k35hj243j5h2kl34h54k"
},
"example #4": {
"value": "prefix-gibberish-afterwards"
}
},
"x-sample-entry": "000000000000000000000019"
}
]
},
"/param-patterns/{parampatternId}/1": {
"get": {
"tags": [
"param-patterns"
],
"summary": "/param-patterns/{parampatternId}/1",
"description": "Kubeshark observed 1 entries (0 failed), at 0.00 hits/s, average response time is 0.00 seconds",
"operationId": "<UUID4>",
"responses": {
"200": {
"description": "Successful call with status 200",
"content": {
"": {
"x-sample-entry": "000000000000000000000015"
}
},
"x-sample-entry": "000000000000000000000015"
}
},
"x-counters-per-source": {
"": {
"entries": 1,
"failures": 0,
"firstSeen": 1567750582.00,
"lastSeen": 1567750582.00,
"sumRT": 0.00,
"sumDuration": 0
}
},
"x-counters-total": {
"entries": 1,
"failures": 0,
"firstSeen": 1567750582.00,
"lastSeen": 1567750582.00,
"sumRT": 0.00,
"sumDuration": 0
},
"x-last-seen-ts": 1567750582.00,
"x-sample-entry": "000000000000000000000015"
},
"parameters": [
{
"name": "parampatternId",
"in": "path",
"required": true,
"style": "simple",
"schema": {
"type": "string",
"pattern": "^prefix-gibberish-.+"
},
"examples": {
"example #0": {
"value": "prefix-gibberish-sfdlasdfkadf87sd93284q24r"
},
"example #1": {
"value": "prefix-gibberish-adslkfasdf89sa7dfasddafa8a98sd7kansdf"
},
"example #2": {
"value": "prefix-gibberish-4jk5l2345h2452l4352435jlk45"
},
"example #3": {
"value": "prefix-gibberish-84395h2j4k35hj243j5h2kl34h54k"
},
"example #4": {
"value": "prefix-gibberish-afterwards"
}
},
"x-sample-entry": "000000000000000000000019"
}
]
},
"/param-patterns/{parampatternId}/static": {
"get": {
"tags": [
"param-patterns"
],
"summary": "/param-patterns/{parampatternId}/static",
"description": "Kubeshark observed 1 entries (0 failed), at 0.00 hits/s, average response time is 0.00 seconds",
"operationId": "<UUID4>",
"responses": {
"200": {
"description": "Successful call with status 200",
"content": {
"": {
"x-sample-entry": "000000000000000000000016"
}
},
"x-sample-entry": "000000000000000000000016"
}
},
"x-counters-per-source": {
"": {
"entries": 1,
"failures": 0,
"firstSeen": 1567750582.00,
"lastSeen": 1567750582.00,
"sumRT": 0.00,
"sumDuration": 0
}
},
"x-counters-total": {
"entries": 1,
"failures": 0,
"firstSeen": 1567750582.00,
"lastSeen": 1567750582.00,
"sumRT": 0.00,
"sumDuration": 0
},
"x-last-seen-ts": 1567750582.00,
"x-sample-entry": "000000000000000000000016"
},
"parameters": [
{
"name": "parampatternId",
"in": "path",
"required": true,
"style": "simple",
"schema": {
"type": "string",
"pattern": "^prefix-gibberish-.+"
},
"examples": {
"example #0": {
"value": "prefix-gibberish-sfdlasdfkadf87sd93284q24r"
},
"example #1": {
"value": "prefix-gibberish-adslkfasdf89sa7dfasddafa8a98sd7kansdf"
},
"example #2": {
"value": "prefix-gibberish-4jk5l2345h2452l4352435jlk45"
},
"example #3": {
"value": "prefix-gibberish-84395h2j4k35hj243j5h2kl34h54k"
},
"example #4": {
"value": "prefix-gibberish-afterwards"
}
},
"x-sample-entry": "000000000000000000000019"
}
]
},
"/param-patterns/{parampatternId}/{param1}": {
"get": {
"tags": [
"param-patterns"
],
"summary": "/param-patterns/{parampatternId}/{param1}",
"description": "Kubeshark observed 1 entries (0 failed), at 0.00 hits/s, average response time is 0.00 seconds",
"operationId": "<UUID4>",
"responses": {
"200": {
"description": "Successful call with status 200",
"content": {
"": {
"x-sample-entry": "000000000000000000000019"
}
},
"x-sample-entry": "000000000000000000000019"
}
},
"x-counters-per-source": {
"": {
"entries": 1,
"failures": 0,
"firstSeen": 1567750582.00,
"lastSeen": 1567750582.00,
"sumRT": 0.00,
"sumDuration": 0
}
},
"x-counters-total": {
"entries": 1,
"failures": 0,
"firstSeen": 1567750582.00,
"lastSeen": 1567750582.00,
"sumRT": 0.00,
"sumDuration": 0
},
"x-last-seen-ts": 1567750582.00,
"x-sample-entry": "000000000000000000000019"
},
"parameters": [
{
"name": "param1",
"in": "path",
"required": true,
"style": "simple",
"schema": {
"type": "string"
},
"examples": {
"example #0": {
"value": "23421"
}
},
"x-sample-entry": "000000000000000000000019"
},
{
"name": "parampatternId",
"in": "path",
"required": true,
"style": "simple",
"schema": {
"type": "string",
"pattern": "^prefix-gibberish-.+"
},
"examples": {
"example #0": {
"value": "prefix-gibberish-sfdlasdfkadf87sd93284q24r"
},
"example #1": {
"value": "prefix-gibberish-adslkfasdf89sa7dfasddafa8a98sd7kansdf"
},
"example #2": {
"value": "prefix-gibberish-4jk5l2345h2452l4352435jlk45"
},
"example #3": {
"value": "prefix-gibberish-84395h2j4k35hj243j5h2kl34h54k"
},
"example #4": {
"value": "prefix-gibberish-afterwards"
}
},
"x-sample-entry": "000000000000000000000019"
}
]
},
"/{Id}": {
"get": {
"summary": "/{Id}",
"description": "Kubeshark observed 1 entries (0 failed), at 0.00 hits/s, average response time is 0.63 seconds",
"operationId": "<UUID4>",
"responses": {
"200": {
"description": "Successful call with status 200",
"content": {
"application/json": {
"example": null,
"x-sample-entry": "000000000000000000000003"
}
},
"x-sample-entry": "000000000000000000000003"
}
},
"x-counters-per-source": {
"": {
"entries": 1,
"failures": 0,
"firstSeen": 1567750579.74,
"lastSeen": 1567750579.74,
"sumRT": 0.63,
"sumDuration": 0
}
},
"x-counters-total": {
"entries": 1,
"failures": 0,
"firstSeen": 1567750579.74,
"lastSeen": 1567750579.74,
"sumRT": 0.63,
"sumDuration": 0
},
"x-last-seen-ts": 1567750579.74,
"x-sample-entry": "000000000000000000000003"
},
"parameters": [
{
"name": "Id",
"in": "path",
"required": true,
"style": "simple",
"schema": {
"type": "string"
},
"examples": {
"example #0": {
"value": "<UUID4>"
},
"example #1": {
"value": "<UUID4>"
}
},
"x-sample-entry": "000000000000000000000003"
}
]
},
"/{Id}/sub1": {
"get": {
"summary": "/{Id}/sub1",
"description": "Kubeshark observed 1 entries (0 failed), at 0.00 hits/s, average response time is 0.11 seconds",
"operationId": "<UUID4>",
"responses": {
"200": {
"description": "Successful call with status 200",
"content": {
"text/html": {
"x-sample-entry": "000000000000000000000001"
}
},
"x-sample-entry": "000000000000000000000001"
}
},
"x-counters-per-source": {
"": {
"entries": 1,
"failures": 0,
"firstSeen": 1567750483.86,
"lastSeen": 1567750483.86,
"sumRT": 0.11,
"sumDuration": 0
}
},
"x-counters-total": {
"entries": 1,
"failures": 0,
"firstSeen": 1567750483.86,
"lastSeen": 1567750483.86,
"sumRT": 0.11,
"sumDuration": 0
},
"x-last-seen-ts": 1567750483.86,
"x-sample-entry": "000000000000000000000001"
},
"parameters": [
{
"name": "Id",
"in": "path",
"required": true,
"style": "simple",
"schema": {
"type": "string"
},
"examples": {
"example #0": {
"value": "<UUID4>"
},
"example #1": {
"value": "<UUID4>"
}
},
"x-sample-entry": "000000000000000000000003"
}
]
},
"/{Id}/sub2": {
"get": {
"summary": "/{Id}/sub2",
"description": "Kubeshark observed 1 entries (0 failed), at 0.00 hits/s, average response time is 0.63 seconds",
"operationId": "<UUID4>",
"responses": {
"200": {
"description": "Successful call with status 200",
"content": {
"application/json": {
"example": null,
"x-sample-entry": "000000000000000000000002"
}
},
"x-sample-entry": "000000000000000000000002"
}
},
"x-counters-per-source": {
"": {
"entries": 1,
"failures": 0,
"firstSeen": 1567750578.74,
"lastSeen": 1567750578.74,
"sumRT": 0.63,
"sumDuration": 0
}
},
"x-counters-total": {
"entries": 1,
"failures": 0,
"firstSeen": 1567750578.74,
"lastSeen": 1567750578.74,
"sumRT": 0.63,
"sumDuration": 0
},
"x-last-seen-ts": 1567750578.74,
"x-sample-entry": "000000000000000000000002"
},
"parameters": [
{
"name": "Id",
"in": "path",
"required": true,
"style": "simple",
"schema": {
"type": "string"
},
"examples": {
"example #0": {
"value": "<UUID4>"
},
"example #1": {
"value": "<UUID4>"
}
},
"x-sample-entry": "000000000000000000000003"
}
]
}
},
"x-counters-per-source": {
"": {
"entries": 19,
"failures": 0,
"firstSeen": 1567750483.86,
"lastSeen": 1567750582.74,
"sumRT": 3.27,
"sumDuration": 2.01
}
},
"x-counters-total": {
"entries": 19,
"failures": 0,
"firstSeen": 1567750483.86,
"lastSeen": 1567750582.74,
"sumRT": 3.27,
"sumDuration": 2.01
}
}

50
agent/pkg/oas/test_artifacts/trcc.json
View File

@ -1,50 +0,0 @@
{
"openapi": "3.1.0",
"info": {
"title": "Preloaded TRCC",
"version": "0.1",
"description": "Test file for loading pre-existing OAS"
},
"paths": {
"/models/{id}": {
"parameters": [
{
"name": "id",
"in": "path",
"required": true,
"style": "simple",
"schema": {
"type": "string",
"pattern": ".+(_|-|\\.).+"
},
"example": "some-uuid-maybe"
}
]
},
"/models/{id}/{id2}": {
"parameters": [
{
"name": "id",
"in": "path",
"required": true,
"style": "simple",
"schema": {
"type": "string",
"pattern": ".+(_|-|\\.).+"
},
"example": "some-uuid-maybe"
},
{
"name": "id2",
"in": "path",
"required": true,
"style": "simple",
"schema": {
"type": "string",
"pattern": "\\d+"
}
}
]
}
}
}

313
agent/pkg/oas/tree.go
View File

@ -1,313 +0,0 @@
package oas
import (
"encoding/json"
"net/url"
"regexp"
"strconv"
"strings"
"github.com/chanced/openapi"
"github.com/kubeshark/kubeshark/logger"
)
type NodePath = []string
type Node struct {
constant *string
pathParam *openapi.ParameterObj
pathObj *openapi.PathObj
parent *Node
children []*Node
}
func (n *Node) getOrSet(path NodePath, existingPathObj *openapi.PathObj, sampleId string) (node *Node) {
if existingPathObj == nil {
panic("Invalid function call")
}
pathChunk := path[0]
potentialMatrix := strings.SplitN(pathChunk, ";", 2)
if len(potentialMatrix) > 1 {
pathChunk = potentialMatrix[0]
logger.Log.Warningf("URI matrix params are not supported: %s", potentialMatrix[1])
}
chunkIsParam := strings.HasPrefix(pathChunk, "{") && strings.HasSuffix(pathChunk, "}")
pathChunk, err := url.PathUnescape(pathChunk)
if err != nil {
logger.Log.Warningf("URI segment is not correctly encoded: %s", pathChunk)
// any side effects on continuing?
}
chunkIsGibberish := IsGibberish(pathChunk) && !IsVersionString(pathChunk)
var paramObj *openapi.ParameterObj
if chunkIsParam && existingPathObj != nil && existingPathObj.Parameters != nil {
_, paramObj = findParamByName(existingPathObj.Parameters, openapi.InPath, pathChunk[1:len(pathChunk)-1])
}
if paramObj == nil {
node = n.searchInConstants(pathChunk)
}
if node == nil && pathChunk != "" {
node = n.searchInParams(paramObj, pathChunk, chunkIsGibberish)
}
// still no node found, should create it
if node == nil {
node = new(Node)
node.parent = n
n.children = append(n.children, node)
if paramObj != nil {
node.pathParam = paramObj
} else if chunkIsGibberish {
newParam := n.createParam()
node.pathParam = newParam
} else {
node.constant = &pathChunk
}
}
if node.pathParam != nil {
setSampleID(&node.pathParam.Extensions, sampleId)
}
// add example if it's a gibberish chunk
if node.pathParam != nil && !chunkIsParam {
exmp := &node.pathParam.Examples
err := fillParamExample(&exmp, pathChunk)
if err != nil {
logger.Log.Warningf("Failed to add example to a parameter: %s", err)
}
if len(*exmp) >= 3 && node.pathParam.Schema.Pattern == nil { // is it enough to decide on 2 samples?
node.pathParam.Schema.Pattern = getPatternFromExamples(exmp)
}
}
// TODO: eat up trailing slash, in a smart way: node.pathObj!=nil && path[1]==""
if len(path) > 1 {
return node.getOrSet(path[1:], existingPathObj, sampleId)
} else if node.pathObj == nil {
node.pathObj = existingPathObj
}
return node
}
func getPatternFromExamples(exmp *openapi.Examples) *openapi.Regexp {
allInts := true
strs := make([]string, 0)
for _, example := range *exmp {
exampleObj, err := example.ResolveExample(exampleResolver)
if err != nil {
continue
}
var value string
err = json.Unmarshal(exampleObj.Value, &value)
if err != nil {
logger.Log.Warningf("Failed decoding parameter example into string: %s", err)
continue
}
strs = append(strs, value)
if _, err := strconv.Atoi(value); err != nil {
allInts = false
}
}
if allInts {
re := new(openapi.Regexp)
re.Regexp = regexp.MustCompile(`\d+`)
return re
} else {
prefix := longestCommonXfixStr(strs, true)
suffix := longestCommonXfixStr(strs, false)
pat := ""
separators := "-._/:|*,+" // TODO: we could also cut prefix till the last separator
if len(prefix) > 0 && strings.Contains(separators, string(prefix[len(prefix)-1])) {
pat = "^" + regexp.QuoteMeta(prefix)
}
pat += ".+"
if len(suffix) > 0 && strings.Contains(separators, string(suffix[0])) {
pat += regexp.QuoteMeta(suffix) + "$"
}
if pat != ".+" {
re := new(openapi.Regexp)
re.Regexp = regexp.MustCompile(pat)
return re
}
}
return nil
}
func (n *Node) createParam() *openapi.ParameterObj {
name := "param"
if n.constant != nil { // the node is already a param
// REST assumption, not always correct
if strings.HasSuffix(*n.constant, "es") && len(*n.constant) > 4 {
name = *n.constant
name = name[:len(name)-2] + "Id"
} else if strings.HasSuffix(*n.constant, "s") && len(*n.constant) > 3 {
name = *n.constant
name = name[:len(name)-1] + "Id"
} else {
name = *n.constant + "Id"
}
name = cleanStr(name, isAlNumRune)
if !isAlphaRune(rune(name[0])) {
name = "_" + name
}
}
newParam := createSimpleParam(name, "path", "string")
x := n.countParentParams()
if x > 0 {
newParam.Name = newParam.Name + strconv.Itoa(x)
}
return newParam
}
func (n *Node) searchInParams(paramObj *openapi.ParameterObj, chunk string, chunkIsGibberish bool) *Node {
// look among params
for _, subnode := range n.children {
if subnode.constant != nil {
continue
}
if paramObj != nil {
// TODO: mergeParam(subnode.pathParam, paramObj)
return subnode
} else if subnode.pathParam.Schema.Pattern != nil { // it has defined param pattern, have to respect it
// TODO: and not in exceptions
if subnode.pathParam.Schema.Pattern.Match([]byte(chunk)) {
return subnode
} else if chunkIsGibberish {
// TODO: what to do if gibberish chunk does not match the pattern and not in exceptions?
return nil
} else {
return nil
}
} else if chunkIsGibberish {
return subnode
}
}
return nil
}
func (n *Node) searchInConstants(pathChunk string) *Node {
// look among constants
for _, subnode := range n.children {
if subnode.constant == nil {
continue
}
if *subnode.constant == pathChunk {
return subnode
}
}
return nil
}
func (n *Node) compact() {
// TODO
}
func (n *Node) listPaths() *openapi.Paths {
paths := &openapi.Paths{Items: map[openapi.PathValue]*openapi.PathObj{}}
var strChunk string
if n.constant != nil {
strChunk = *n.constant
} else if n.pathParam != nil {
strChunk = "{" + n.pathParam.Name + "}"
} // else -> this is the root node
// add self
if n.pathObj != nil {
fillPathParams(n, n.pathObj)
paths.Items[openapi.PathValue(strChunk)] = n.pathObj
}
// recurse into children
for _, child := range n.children {
subPaths := child.listPaths()
for path, pathObj := range subPaths.Items {
var concat string
if n.parent == nil {
concat = string(path)
} else {
concat = strChunk + "/" + string(path)
}
paths.Items[openapi.PathValue(concat)] = pathObj
}
}
return paths
}
func fillPathParams(n *Node, pathObj *openapi.PathObj) {
// collect all path parameters from parent hierarchy
node := n
for {
if node.pathParam != nil {
initParams(&pathObj.Parameters)
idx, paramObj := findParamByName(pathObj.Parameters, openapi.InPath, node.pathParam.Name)
if paramObj == nil {
appended := append(*pathObj.Parameters, node.pathParam)
pathObj.Parameters = &appended
} else {
(*pathObj.Parameters)[idx] = paramObj
}
}
node = node.parent
if node == nil {
break
}
}
}
type PathAndOp struct {
path string
op *openapi.Operation
}
func (n *Node) listOps() []PathAndOp {
res := make([]PathAndOp, 0)
for path, pathObj := range n.listPaths().Items {
for _, op := range getOps(pathObj) {
res = append(res, PathAndOp{path: string(path), op: op})
}
}
return res
}
func (n *Node) countParentParams() int {
res := 0
node := n
for {
if node.pathParam != nil {
res++
}
if node.parent == nil {
break
}
node = node.parent
}
return res
}

39
agent/pkg/oas/tree_test.go
View File

@ -1,39 +0,0 @@
package oas
import (
"fmt"
"strings"
"testing"
"github.com/chanced/openapi"
)
func TestTree(t *testing.T) {
testCases := []struct {
inp string
numParams int
label string
}{
{"/", 0, ""},
{"/v1.0.0/config/launcher/sp_nKNHCzsN/f34efcae-6583-11eb-908a-00b0fcb9d4f6/vendor,init,conversation", 1, "vendor,init,conversation"},
{"/v1.0.0/config/launcher/sp_nKNHCzsN/{f34efcae-6583-11eb-908a-00b0fcb9d4f6}/vendor,init,conversation", 0, "vendor,init,conversation"},
{"/getSvgs/size/small/brand/SFLY/layoutId/170943/layoutVersion/1/sizeId/742/surface/0/isLandscape/true/childSkus/%7B%7D", 1, "{}"},
}
tree := new(Node)
for i, tc := range testCases {
split := strings.Split(tc.inp, "/")
pathObj := new(openapi.PathObj)
node := tree.getOrSet(split, pathObj, fmt.Sprintf("%024d", i))
fillPathParams(node, pathObj)
if node.constant != nil && *node.constant != tc.label {
t.Errorf("Constant does not match: %s != %s", *node.constant, tc.label)
}
if tc.numParams > 0 && (pathObj.Parameters == nil || len(*pathObj.Parameters) < tc.numParams) {
t.Errorf("Wrong num of params, expected: %d", tc.numParams)
}
}
}

7707
agent/pkg/oas/trigrams.go
View File

File diff suppressed because it is too large Load Diff

490
agent/pkg/oas/utils.go
View File

@ -1,490 +0,0 @@
package oas
import (
"encoding/json"
"errors"
"strconv"
"strings"
"github.com/kubeshark/kubeshark/agent/pkg/har"
"github.com/chanced/openapi"
"github.com/kubeshark/kubeshark/logger"
)
func exampleResolver(ref string) (*openapi.ExampleObj, error) {
return nil, errors.New("JSON references are not supported at the moment: " + ref)
}
func responseResolver(ref string) (*openapi.ResponseObj, error) {
return nil, errors.New("JSON references are not supported at the moment: " + ref)
}
func reqBodyResolver(ref string) (*openapi.RequestBodyObj, error) {
return nil, errors.New("JSON references are not supported at the moment: " + ref)
}
func paramResolver(ref string) (*openapi.ParameterObj, error) {
return nil, errors.New("JSON references are not supported at the moment: " + ref)
}
func headerResolver(ref string) (*openapi.HeaderObj, error) {
return nil, errors.New("JSON references are not supported at the moment: " + ref)
}
func initParams(obj **openapi.ParameterList) {
if *obj == nil {
var params openapi.ParameterList = make([]openapi.Parameter, 0)
*obj = &params
}
}
func initHeaders(respObj *openapi.ResponseObj) {
if respObj.Headers == nil {
var created openapi.Headers = map[string]openapi.Header{}
respObj.Headers = created
}
}
func createSimpleParam(name string, in openapi.In, ptype openapi.SchemaType) *openapi.ParameterObj {
if name == "" {
panic("Cannot create parameter with empty name")
}
required := true // FFS! https://stackoverflow.com/questions/32364027/reference-a-boolean-for-assignment-in-a-struct/32364093
schema := new(openapi.SchemaObj)
schema.Type = openapi.Types{ptype}
style := openapi.StyleSimple
if in == openapi.InQuery {
style = openapi.StyleForm
}
newParam := openapi.ParameterObj{
Name: name,
In: in,
Style: string(style),
Examples: map[string]openapi.Example{},
Schema: schema,
Required: &required,
}
return &newParam
}
func findParamByName(params *openapi.ParameterList, in openapi.In, name string) (idx int, pathParam *openapi.ParameterObj) {
caseInsensitive := in == openapi.InHeader
for i, param := range *params {
idx = i
paramObj, err := param.ResolveParameter(paramResolver)
if err != nil {
logger.Log.Warningf("Failed to resolve reference: %s", err)
continue
}
if paramObj.In != in {
continue
}
if paramObj.Name == name || (caseInsensitive && strings.EqualFold(paramObj.Name, name)) {
pathParam = paramObj
break
}
}
return idx, pathParam
}
func findHeaderByName(headers *openapi.Headers, name string) *openapi.HeaderObj {
for hname, param := range *headers {
hdrObj, err := param.ResolveHeader(headerResolver)
if err != nil {
logger.Log.Warningf("Failed to resolve reference: %s", err)
continue
}
if strings.EqualFold(hname, name) {
return hdrObj
}
}
return nil
}
type nvParams struct {
In openapi.In
Pairs []har.NVP
IsIgnored func(name string) bool
GeneralizeName func(name string) string
}
func handleNameVals(gw nvParams, params **openapi.ParameterList, checkIgnore bool, sampleId string) {
visited := map[string]*openapi.ParameterObj{}
for _, pair := range gw.Pairs {
if (checkIgnore && gw.IsIgnored(pair.Name)) || pair.Name == "" {
continue
}
nameGeneral := gw.GeneralizeName(pair.Name)
initParams(params)
_, param := findParamByName(*params, gw.In, pair.Name)
if param == nil {
param = createSimpleParam(nameGeneral, gw.In, openapi.TypeString)
appended := append(**params, param)
*params = &appended
}
exmp := &param.Examples
err := fillParamExample(&exmp, pair.Value)
if err != nil {
logger.Log.Warningf("Failed to add example to a parameter: %s", err)
}
visited[nameGeneral] = param
setSampleID(&param.Extensions, sampleId)
}
// maintain "required" flag
if *params != nil {
for _, param := range **params {
paramObj, err := param.ResolveParameter(paramResolver)
if err != nil {
logger.Log.Warningf("Failed to resolve param: %s", err)
continue
}
if paramObj.In != gw.In {
continue
}
_, ok := visited[strings.ToLower(paramObj.Name)]
if !ok {
flag := false
paramObj.Required = &flag
}
}
}
}
func createHeader(ptype openapi.SchemaType) *openapi.HeaderObj {
required := true // FFS! https://stackoverflow.com/questions/32364027/reference-a-boolean-for-assignment-in-a-struct/32364093
schema := new(openapi.SchemaObj)
schema.Type = make(openapi.Types, 0)
schema.Type = append(schema.Type, ptype)
style := openapi.StyleSimple
newParam := openapi.HeaderObj{
Style: string(style),
Examples: map[string]openapi.Example{},
Schema: schema,
Required: &required,
}
return &newParam
}
func fillParamExample(param **openapi.Examples, exampleValue string) error {
if **param == nil {
**param = map[string]openapi.Example{}
}
cnt := 0
for _, example := range **param {
cnt++
exampleObj, err := example.ResolveExample(exampleResolver)
if err != nil {
continue
}
var value string
err = json.Unmarshal(exampleObj.Value, &value)
if err != nil {
logger.Log.Warningf("Failed decoding parameter example into string: %s", err)
continue
}
if value == exampleValue || cnt >= 5 { // 5 examples is enough
return nil
}
}
valMsg, err := json.Marshal(exampleValue)
if err != nil {
return err
}
themap := **param
themap["example #"+strconv.Itoa(cnt)] = &openapi.ExampleObj{Value: valMsg}
return nil
}
// TODO: somehow generalize the two example setting functions, plus add body example handling
func addSchemaExample(existing *openapi.SchemaObj, bodyStr string) {
if len(existing.Examples) < 5 {
found := false
for _, eVal := range existing.Examples {
existingExample := ""
err := json.Unmarshal(eVal, &existingExample)
if err != nil {
logger.Log.Debugf("Failed to unmarshal example: %v", eVal)
continue
}
if existingExample == bodyStr {
found = true
break
}
}
if !found {
example, err := json.Marshal(bodyStr)
if err != nil {
logger.Log.Debugf("Failed to marshal example: %v", bodyStr)
return
}
existing.Examples = append(existing.Examples, example)
}
}
}
func longestCommonXfix(strs [][]string, pre bool) []string { // https://github.com/jpillora/longestcommon
empty := make([]string, 0)
//short-circuit empty list
if len(strs) == 0 {
return empty
}
xfix := strs[0]
//short-circuit single-element list
if len(strs) == 1 {
return xfix
}
//compare first to rest
for _, str := range strs[1:] {
xfixl := len(xfix)
strl := len(str)
//short-circuit empty strings
if xfixl == 0 || strl == 0 {
return empty
}
//maximum possible length
maxl := xfixl
if strl < maxl {
maxl = strl
}
//compare letters
if pre {
//prefix, iterate left to right
for i := 0; i < maxl; i++ {
if xfix[i] != str[i] {
xfix = xfix[:i]
break
}
}
} else {
//suffix, iternate right to left
for i := 0; i < maxl; i++ {
xi := xfixl - i - 1
si := strl - i - 1
if xfix[xi] != str[si] {
xfix = xfix[xi+1:]
break
}
}
}
}
return xfix
}
func longestCommonXfixStr(strs []string, pre bool) string { // https://github.com/jpillora/longestcommon
//short-circuit empty list
if len(strs) == 0 {
return ""
}
xfix := strs[0]
//short-circuit single-element list
if len(strs) == 1 {
return xfix
}
//compare first to rest
for _, str := range strs[1:] {
xfixl := len(xfix)
strl := len(str)
//short-circuit empty strings
if xfixl == 0 || strl == 0 {
return ""
}
//maximum possible length
maxl := xfixl
if strl < maxl {
maxl = strl
}
//compare letters
if pre {
//prefix, iterate left to right
for i := 0; i < maxl; i++ {
if xfix[i] != str[i] {
xfix = xfix[:i]
break
}
}
} else {
//suffix, iternate right to left
for i := 0; i < maxl; i++ {
xi := xfixl - i - 1
si := strl - i - 1
if xfix[xi] != str[si] {
xfix = xfix[xi+1:]
break
}
}
}
}
return xfix
}
func getSimilarPrefix(strs []string) string {
chunked := make([][]string, 0)
for _, item := range strs {
chunked = append(chunked, strings.Split(item, "/"))
}
cmn := longestCommonXfix(chunked, true)
res := make([]string, 0)
for _, chunk := range cmn {
if chunk != "api" && !IsVersionString(chunk) && !strings.HasPrefix(chunk, "{") {
res = append(res, chunk)
}
}
return strings.Join(res[1:], ".")
}
// returns all non-nil ops in PathObj
func getOps(pathObj *openapi.PathObj) []*openapi.Operation {
ops := []**openapi.Operation{&pathObj.Get, &pathObj.Patch, &pathObj.Put, &pathObj.Options, &pathObj.Post, &pathObj.Trace, &pathObj.Head, &pathObj.Delete}
res := make([]*openapi.Operation, 0)
for _, opp := range ops {
if *opp == nil {
continue
}
res = append(res, *opp)
}
return res
}
// parses JSON into any possible value
func anyJSON(text string) (anyVal interface{}, isJSON bool) {
isJSON = true
asMap := map[string]interface{}{}
err := json.Unmarshal([]byte(text), &asMap)
if err == nil && asMap != nil {
return asMap, isJSON
}
asArray := make([]interface{}, 0)
err = json.Unmarshal([]byte(text), &asArray)
if err == nil && asArray != nil {
return asArray, isJSON
}
asString := ""
sPtr := &asString
err = json.Unmarshal([]byte(text), &sPtr)
if err == nil && sPtr != nil {
return asString, isJSON
}
asInt := 0
intPtr := &asInt
err = json.Unmarshal([]byte(text), &intPtr)
if err == nil && intPtr != nil {
return asInt, isJSON
}
asFloat := 0.0
floatPtr := &asFloat
err = json.Unmarshal([]byte(text), &floatPtr)
if err == nil && floatPtr != nil {
return asFloat, isJSON
}
asBool := false
boolPtr := &asBool
err = json.Unmarshal([]byte(text), &boolPtr)
if err == nil && boolPtr != nil {
return asBool, isJSON
}
if text == "null" {
return nil, isJSON
}
return nil, false
}
func cleanStr(str string, criterion func(r rune) bool) string {
s := []byte(str)
j := 0
for _, b := range s {
if criterion(rune(b)) {
s[j] = b
j++
}
}
return string(s[:j])
}
/*
func isAlpha(s string) bool {
for _, r := range s {
if isAlphaRune(r) {
return false
}
}
return true
}
*/
func isAlphaRune(r rune) bool {
return !((r < 'a' || r > 'z') && (r < 'A' || r > 'Z'))
}
func isAlNumRune(b rune) bool {
return isAlphaRune(b) || ('0' <= b && b <= '9')
}
func deleteFromSlice(s []string, val string) []string {
temp := s[:0]
for _, x := range s {
if x != val {
temp = append(temp, x)
}
}
return temp
}
func sliceContains(s []string, e string) bool {
for _, a := range s {
if a == e {
return true
}
}
return false
}
func intersectSliceWithMap(required []string, names map[string]struct{}) []string {
for name := range names {
if !sliceContains(required, name) {
required = deleteFromSlice(required, name)
}
}
return required
}
func setSampleID(extensions *openapi.Extensions, id string) {
if id != "" {
if *extensions == nil {
*extensions = openapi.Extensions{}
}
err := (extensions).SetExtension(SampleId, id)
if err != nil {
logger.Log.Warningf("Failed to set sample ID: %s", err)
}
}
}

59
agent/pkg/oas/utils_test.go
View File

@ -1,59 +0,0 @@
package oas
import (
"testing"
)
func TestAnyJSON(t *testing.T) {
testCases := []struct {
inp string
isJSON bool
out interface{}
}{
{`{"key": 1, "keyNull": null}`, true, nil},
{`[{"key": "val"}, ["subarray"], "string", 1, 2.2, true, null]`, true, nil},
{`"somestring"`, true, "somestring"},
{"0", true, 0},
{"0.5", true, 0.5},
{"true", true, true},
{"null", true, nil},
{"sabbra cadabra", false, nil},
{"0.1.2.3", false, nil},
}
for _, tc := range testCases {
any, isJSON := anyJSON(tc.inp)
if isJSON != tc.isJSON {
t.Errorf("Parse flag mismatch: %t != %t", tc.isJSON, isJSON)
} else if isJSON && tc.out != nil && tc.out != any {
t.Errorf("%s != %s", any, tc.out)
} else if tc.inp == "null" && any != nil {
t.Errorf("null has to parse as nil (but got %s)", any)
} else {
t.Logf("%s => %v", tc.inp, any)
}
}
}
func TestStrRunes(t *testing.T) {
if isAlphaRune('5') {
t.Logf("Failed")
}
if !isAlphaRune('a') {
t.Logf("Failed")
}
if !isAlNumRune('5') {
t.Logf("Failed")
}
if isAlNumRune(' ') {
t.Logf("Failed")
}
if cleanStr("-abc_567", isAlphaRune) != "abc" {
t.Logf("Failed")
}
if cleanStr("-abc_567", isAlNumRune) != "abc567" {
t.Logf("Failed")
}
}

366
agent/pkg/providers/stats_provider.go
View File

@ -1,366 +0,0 @@
package providers
import (
"crypto/md5"
"encoding/hex"
"fmt"
"reflect"
"sync"
"time"
"github.com/jinzhu/copier"
"github.com/kubeshark/kubeshark/logger"
"github.com/kubeshark/worker/api"
)
type GeneralStats struct {
EntriesCount int
EntriesVolumeInGB float64
FirstEntryTimestamp int
LastEntryTimestamp int
}
type BucketStats []*TimeFrameStatsValue
type TimeFrameStatsValue struct {
BucketTime time.Time `json:"timestamp"`
ProtocolStats map[string]ProtocolStats `json:"protocols"`
}
type ProtocolStats struct {
MethodsStats map[string]*SizeAndEntriesCount `json:"methods"`
}
type SizeAndEntriesCount struct {
EntriesCount int `json:"entriesCount"`
VolumeInBytes int `json:"volumeInBytes"`
}
type AccumulativeStatsCounter struct {
Name string `json:"name"`
Color string `json:"color"`
EntriesCount int `json:"entriesCount"`
VolumeSizeBytes int `json:"volumeSizeBytes"`
}
type AccumulativeStatsProtocol struct {
AccumulativeStatsCounter
Methods []*AccumulativeStatsCounter `json:"methods"`
}
type AccumulativeStatsProtocolTime struct {
ProtocolsData []*AccumulativeStatsProtocol `json:"protocols"`
Time int64 `json:"timestamp"`
}
type TrafficStatsResponse struct {
Protocols []string `json:"protocols"`
PieStats []*AccumulativeStatsProtocol `json:"pie"`
TimelineStats []*AccumulativeStatsProtocolTime `json:"timeline"`
}
var (
generalStats = GeneralStats{}
bucketsStats = BucketStats{}
bucketStatsLocker = sync.Mutex{}
protocolToColor = map[string]string{}
)
const (
InternalBucketThreshold = time.Minute * 1
MaxNumberOfBars = 30
)
func ResetGeneralStats() {
generalStats = GeneralStats{}
}
func GetGeneralStats() *GeneralStats {
return &generalStats
}
func InitProtocolToColor(protocolMap map[string]*api.Protocol) {
for item, value := range protocolMap {
protocolToColor[api.GetProtocolSummary(item).Abbreviation] = value.BackgroundColor
}
}
func GetTrafficStats(startTime time.Time, endTime time.Time) *TrafficStatsResponse {
bucketsStatsCopy := getFilteredBucketStatsCopy(startTime, endTime)
return &TrafficStatsResponse{
Protocols: getAvailableProtocols(bucketsStatsCopy),
PieStats: getAccumulativeStats(bucketsStatsCopy),
TimelineStats: getAccumulativeStatsTiming(bucketsStatsCopy),
}
}
func EntryAdded(size int, summery *api.BaseEntry) {
generalStats.EntriesCount++
generalStats.EntriesVolumeInGB += float64(size) / (1 << 30)
currentTimestamp := int(time.Now().Unix())
if reflect.Value.IsZero(reflect.ValueOf(generalStats.FirstEntryTimestamp)) {
generalStats.FirstEntryTimestamp = currentTimestamp
}
addToBucketStats(size, summery)
generalStats.LastEntryTimestamp = currentTimestamp
}
func calculateInterval(firstTimestamp int64, lastTimestamp int64) time.Duration {
validDurations := []time.Duration{
time.Minute,
time.Minute * 2,
time.Minute * 3,
time.Minute * 5,
time.Minute * 10,
time.Minute * 15,
time.Minute * 20,
time.Minute * 30,
time.Minute * 45,
time.Minute * 60,
time.Minute * 75,
time.Minute * 90, // 1.5 minutes
time.Minute * 120, // 2 hours
time.Minute * 150, // 2.5 hours
time.Minute * 180, // 3 hours
time.Minute * 240, // 4 hours
time.Minute * 300, // 5 hours
time.Minute * 360, // 6 hours
time.Minute * 420, // 7 hours
time.Minute * 480, // 8 hours
time.Minute * 540, // 9 hours
time.Minute * 600, // 10 hours
time.Minute * 660, // 11 hours
time.Minute * 720, // 12 hours
time.Minute * 1440, // 24 hours
}
duration := time.Duration(lastTimestamp-firstTimestamp) * time.Second / time.Duration(MaxNumberOfBars)
for _, validDuration := range validDurations {
if validDuration-duration >= 0 {
return validDuration
}
}
return duration.Round(validDurations[len(validDurations)-1])
}
func getAccumulativeStats(stats BucketStats) []*AccumulativeStatsProtocol {
if len(stats) == 0 {
return make([]*AccumulativeStatsProtocol, 0)
}
methodsPerProtocolAggregated := getAggregatedStats(stats)
return convertAccumulativeStatsDictToArray(methodsPerProtocolAggregated)
}
func getAccumulativeStatsTiming(stats BucketStats) []*AccumulativeStatsProtocolTime {
if len(stats) == 0 {
return make([]*AccumulativeStatsProtocolTime, 0)
}
interval := calculateInterval(stats[0].BucketTime.Unix(), stats[len(stats)-1].BucketTime.Unix()) // in seconds
methodsPerProtocolPerTimeAggregated := getAggregatedResultTiming(stats, interval)
return convertAccumulativeStatsTimelineDictToArray(methodsPerProtocolPerTimeAggregated)
}
func addToBucketStats(size int, summery *api.BaseEntry) {
entryTimeBucketRounded := getBucketFromTimeStamp(summery.Timestamp)
if len(bucketsStats) == 0 {
bucketsStats = append(bucketsStats, &TimeFrameStatsValue{
BucketTime: entryTimeBucketRounded,
ProtocolStats: map[string]ProtocolStats{},
})
}
bucketOfEntry := bucketsStats[len(bucketsStats)-1]
if bucketOfEntry.BucketTime != entryTimeBucketRounded {
bucketOfEntry = &TimeFrameStatsValue{
BucketTime: entryTimeBucketRounded,
ProtocolStats: map[string]ProtocolStats{},
}
bucketsStats = append(bucketsStats, bucketOfEntry)
}
if _, found := bucketOfEntry.ProtocolStats[summery.Protocol.Abbreviation]; !found {
bucketOfEntry.ProtocolStats[summery.Protocol.Abbreviation] = ProtocolStats{
MethodsStats: map[string]*SizeAndEntriesCount{},
}
}
if _, found := bucketOfEntry.ProtocolStats[summery.Protocol.Abbreviation].MethodsStats[summery.Method]; !found {
bucketOfEntry.ProtocolStats[summery.Protocol.Abbreviation].MethodsStats[summery.Method] = &SizeAndEntriesCount{
VolumeInBytes: 0,
EntriesCount: 0,
}
}
bucketOfEntry.ProtocolStats[summery.Protocol.Abbreviation].MethodsStats[summery.Method].EntriesCount += 1
bucketOfEntry.ProtocolStats[summery.Protocol.Abbreviation].MethodsStats[summery.Method].VolumeInBytes += size
}
func getBucketFromTimeStamp(timestamp int64) time.Time {
entryTimeStampAsTime := time.UnixMilli(timestamp)
return entryTimeStampAsTime.Add(-1 * InternalBucketThreshold / 2).Round(InternalBucketThreshold)
}
func convertAccumulativeStatsTimelineDictToArray(methodsPerProtocolPerTimeAggregated map[time.Time]map[string]map[string]*AccumulativeStatsCounter) []*AccumulativeStatsProtocolTime {
finalResult := make([]*AccumulativeStatsProtocolTime, 0)
for timeKey, item := range methodsPerProtocolPerTimeAggregated {
protocolsData := make([]*AccumulativeStatsProtocol, 0)
for protocolName, value := range item {
entriesCount := 0
volumeSizeBytes := 0
methods := make([]*AccumulativeStatsCounter, 0)
for _, methodAccData := range value {
entriesCount += methodAccData.EntriesCount
volumeSizeBytes += methodAccData.VolumeSizeBytes
methods = append(methods, methodAccData)
}
protocolsData = append(protocolsData, &AccumulativeStatsProtocol{
AccumulativeStatsCounter: AccumulativeStatsCounter{
Name: protocolName,
Color: protocolToColor[protocolName],
EntriesCount: entriesCount,
VolumeSizeBytes: volumeSizeBytes,
},
Methods: methods,
})
}
finalResult = append(finalResult, &AccumulativeStatsProtocolTime{
Time: timeKey.UnixMilli(),
ProtocolsData: protocolsData,
})
}
return finalResult
}
func convertAccumulativeStatsDictToArray(methodsPerProtocolAggregated map[string]map[string]*AccumulativeStatsCounter) []*AccumulativeStatsProtocol {
protocolsData := make([]*AccumulativeStatsProtocol, 0)
for protocolName, value := range methodsPerProtocolAggregated {
entriesCount := 0
volumeSizeBytes := 0
methods := make([]*AccumulativeStatsCounter, 0)
for _, methodAccData := range value {
entriesCount += methodAccData.EntriesCount
volumeSizeBytes += methodAccData.VolumeSizeBytes
methods = append(methods, methodAccData)
}
protocolsData = append(protocolsData, &AccumulativeStatsProtocol{
AccumulativeStatsCounter: AccumulativeStatsCounter{
Name: protocolName,
Color: protocolToColor[protocolName],
EntriesCount: entriesCount,
VolumeSizeBytes: volumeSizeBytes,
},
Methods: methods,
})
}
return protocolsData
}
func getFilteredBucketStatsCopy(startTime time.Time, endTime time.Time) BucketStats {
bucketStatsCopy := BucketStats{}
bucketStatsLocker.Lock()
if err := copier.Copy(&bucketStatsCopy, bucketsStats); err != nil {
logger.Log.Errorf("Error while copying src stats into temporary copied object")
return nil
}
bucketStatsLocker.Unlock()
filteredBucketStatsCopy := BucketStats{}
interval := InternalBucketThreshold
for _, bucket := range bucketStatsCopy {
if (bucket.BucketTime.After(startTime.Add(-1*interval/2).Round(interval)) && bucket.BucketTime.Before(endTime.Add(-1*interval/2).Round(interval))) ||
bucket.BucketTime.Equal(startTime.Add(-1*interval/2).Round(interval)) ||
bucket.BucketTime.Equal(endTime.Add(-1*interval/2).Round(interval)) {
filteredBucketStatsCopy = append(filteredBucketStatsCopy, bucket)
}
}
return filteredBucketStatsCopy
}
func getAggregatedResultTiming(stats BucketStats, interval time.Duration) map[time.Time]map[string]map[string]*AccumulativeStatsCounter {
methodsPerProtocolPerTimeAggregated := map[time.Time]map[string]map[string]*AccumulativeStatsCounter{}
bucketStatsIndex := len(stats) - 1
for bucketStatsIndex >= 0 {
currentBucketTime := stats[bucketStatsIndex].BucketTime
resultBucketRoundedKey := currentBucketTime.Add(-1 * interval / 2).Round(interval)
for protocolName, data := range stats[bucketStatsIndex].ProtocolStats {
for methodName, dataOfMethod := range data.MethodsStats {
if _, ok := methodsPerProtocolPerTimeAggregated[resultBucketRoundedKey]; !ok {
methodsPerProtocolPerTimeAggregated[resultBucketRoundedKey] = map[string]map[string]*AccumulativeStatsCounter{}
}
if _, ok := methodsPerProtocolPerTimeAggregated[resultBucketRoundedKey][protocolName]; !ok {
methodsPerProtocolPerTimeAggregated[resultBucketRoundedKey][protocolName] = map[string]*AccumulativeStatsCounter{}
}
if _, ok := methodsPerProtocolPerTimeAggregated[resultBucketRoundedKey][protocolName][methodName]; !ok {
methodsPerProtocolPerTimeAggregated[resultBucketRoundedKey][protocolName][methodName] = &AccumulativeStatsCounter{
Name: methodName,
Color: getColorForMethod(protocolName, methodName),
EntriesCount: 0,
VolumeSizeBytes: 0,
}
}
methodsPerProtocolPerTimeAggregated[resultBucketRoundedKey][protocolName][methodName].EntriesCount += dataOfMethod.EntriesCount
methodsPerProtocolPerTimeAggregated[resultBucketRoundedKey][protocolName][methodName].VolumeSizeBytes += dataOfMethod.VolumeInBytes
}
}
bucketStatsIndex--
}
return methodsPerProtocolPerTimeAggregated
}
func getAggregatedStats(stats BucketStats) map[string]map[string]*AccumulativeStatsCounter {
methodsPerProtocolAggregated := make(map[string]map[string]*AccumulativeStatsCounter, 0)
for _, countersOfTimeFrame := range stats {
for protocolName, value := range countersOfTimeFrame.ProtocolStats {
for method, countersValue := range value.MethodsStats {
if _, found := methodsPerProtocolAggregated[protocolName]; !found {
methodsPerProtocolAggregated[protocolName] = map[string]*AccumulativeStatsCounter{}
}
if _, found := methodsPerProtocolAggregated[protocolName][method]; !found {
methodsPerProtocolAggregated[protocolName][method] = &AccumulativeStatsCounter{
Name: method,
Color: getColorForMethod(protocolName, method),
EntriesCount: 0,
VolumeSizeBytes: 0,
}
}
methodsPerProtocolAggregated[protocolName][method].EntriesCount += countersValue.EntriesCount
methodsPerProtocolAggregated[protocolName][method].VolumeSizeBytes += countersValue.VolumeInBytes
}
}
}
return methodsPerProtocolAggregated
}
func getColorForMethod(protocolName string, methodName string) string {
hash := md5.Sum([]byte(fmt.Sprintf("%v_%v", protocolName, methodName)))
input := hex.EncodeToString(hash[:])
return fmt.Sprintf("#%v", input[:6])
}
func getAvailableProtocols(stats BucketStats) []string {
protocols := map[string]bool{}
for _, countersOfTimeFrame := range stats {
for protocolName := range countersOfTimeFrame.ProtocolStats {
protocols[protocolName] = true
}
}
result := make([]string, 0)
for protocol := range protocols {
result = append(result, protocol)
}
result = append(result, "ALL")
return result
}

298
agent/pkg/providers/stats_provider_internal_test.go
View File

@ -1,298 +0,0 @@
package providers
import (
"fmt"
"testing"
"time"
)
func TestGetBucketOfTimeStamp(t *testing.T) {
tests := map[int64]time.Time{
time.Date(2022, time.Month(1), 1, 10, 34, 45, 0, time.Local).UnixMilli(): time.Date(2022, time.Month(1), 1, 10, 34, 00, 0, time.Local),
time.Date(2022, time.Month(1), 1, 10, 34, 00, 0, time.Local).UnixMilli(): time.Date(2022, time.Month(1), 1, 10, 34, 00, 0, time.Local),
time.Date(2022, time.Month(1), 1, 10, 59, 01, 0, time.Local).UnixMilli(): time.Date(2022, time.Month(1), 1, 10, 59, 00, 0, time.Local),
}
for key, value := range tests {
t.Run(fmt.Sprintf("%v", key), func(t *testing.T) {
actual := getBucketFromTimeStamp(key)
if actual != value {
t.Errorf("unexpected result - expected: %v, actual: %v", value, actual)
}
})
}
}
func TestGetAggregatedStatsAllTime(t *testing.T) {
bucketStatsForTest := BucketStats{
&TimeFrameStatsValue{
BucketTime: time.Date(2022, time.Month(1), 1, 10, 00, 00, 0, time.UTC),
ProtocolStats: map[string]ProtocolStats{
"http": {
MethodsStats: map[string]*SizeAndEntriesCount{
"get": {
EntriesCount: 1,
VolumeInBytes: 2,
},
"post": {
EntriesCount: 2,
VolumeInBytes: 3,
},
},
},
"kafka": {
MethodsStats: map[string]*SizeAndEntriesCount{
"listTopics": {
EntriesCount: 5,
VolumeInBytes: 6,
},
},
},
},
},
&TimeFrameStatsValue{
BucketTime: time.Date(2022, time.Month(1), 1, 10, 01, 00, 0, time.UTC),
ProtocolStats: map[string]ProtocolStats{
"http": {
MethodsStats: map[string]*SizeAndEntriesCount{
"get": {
EntriesCount: 1,
VolumeInBytes: 2,
},
"post": {
EntriesCount: 2,
VolumeInBytes: 3,
},
},
},
"redis": {
MethodsStats: map[string]*SizeAndEntriesCount{
"set": {
EntriesCount: 5,
VolumeInBytes: 6,
},
},
},
},
},
}
expected := map[string]map[string]*AccumulativeStatsCounter{
"http": {
"post": {
Name: "post",
EntriesCount: 4,
VolumeSizeBytes: 6,
},
"get": {
Name: "get",
EntriesCount: 2,
VolumeSizeBytes: 4,
},
},
"kafka": {
"listTopics": {
Name: "listTopics",
EntriesCount: 5,
VolumeSizeBytes: 6,
},
},
"redis": {
"set": {
Name: "set",
EntriesCount: 5,
VolumeSizeBytes: 6,
},
},
}
actual := getAggregatedStats(bucketStatsForTest)
if len(actual) != len(expected) {
t.Errorf("unexpected result - expected: %v, actual: %v", len(expected), len(actual))
}
}
func TestGetAggregatedStatsFromSpecificTime(t *testing.T) {
bucketStatsForTest := BucketStats{
&TimeFrameStatsValue{
BucketTime: time.Date(2022, time.Month(1), 1, 10, 00, 00, 0, time.UTC),
ProtocolStats: map[string]ProtocolStats{
"http": {
MethodsStats: map[string]*SizeAndEntriesCount{
"get": {
EntriesCount: 1,
VolumeInBytes: 2,
},
},
},
"kafka": {
MethodsStats: map[string]*SizeAndEntriesCount{
"listTopics": {
EntriesCount: 5,
VolumeInBytes: 6,
},
},
},
},
},
&TimeFrameStatsValue{
BucketTime: time.Date(2022, time.Month(1), 1, 10, 01, 00, 0, time.UTC),
ProtocolStats: map[string]ProtocolStats{
"http": {
MethodsStats: map[string]*SizeAndEntriesCount{
"get": {
EntriesCount: 1,
VolumeInBytes: 2,
},
"post": {
EntriesCount: 2,
VolumeInBytes: 3,
},
},
},
"redis": {
MethodsStats: map[string]*SizeAndEntriesCount{
"set": {
EntriesCount: 5,
VolumeInBytes: 6,
},
},
},
},
},
}
expected := map[time.Time]map[string]map[string]*AccumulativeStatsCounter{
time.Date(2022, time.Month(1), 1, 10, 00, 00, 0, time.UTC): {
"http": {
"post": {
Name: "post",
EntriesCount: 2,
VolumeSizeBytes: 3,
},
"get": {
Name: "get",
EntriesCount: 2,
VolumeSizeBytes: 4,
},
},
"kafka": {
"listTopics": {
Name: "listTopics",
EntriesCount: 5,
VolumeSizeBytes: 6,
},
},
"redis": {
"set": {
Name: "set",
EntriesCount: 5,
VolumeSizeBytes: 6,
},
},
},
}
actual := getAggregatedResultTiming(bucketStatsForTest, time.Minute*5)
if len(actual) != len(expected) {
t.Errorf("unexpected result - expected: %v, actual: %v", len(expected), len(actual))
}
}
func TestGetAggregatedStatsFromSpecificTimeMultipleBuckets(t *testing.T) {
bucketStatsForTest := BucketStats{
&TimeFrameStatsValue{
BucketTime: time.Date(2022, time.Month(1), 1, 10, 00, 00, 0, time.UTC),
ProtocolStats: map[string]ProtocolStats{
"http": {
MethodsStats: map[string]*SizeAndEntriesCount{
"get": {
EntriesCount: 1,
VolumeInBytes: 2,
},
},
},
"kafka": {
MethodsStats: map[string]*SizeAndEntriesCount{
"listTopics": {
EntriesCount: 5,
VolumeInBytes: 6,
},
},
},
},
},
&TimeFrameStatsValue{
BucketTime: time.Date(2022, time.Month(1), 1, 10, 01, 00, 0, time.UTC),
ProtocolStats: map[string]ProtocolStats{
"http": {
MethodsStats: map[string]*SizeAndEntriesCount{
"get": {
EntriesCount: 1,
VolumeInBytes: 2,
},
"post": {
EntriesCount: 2,
VolumeInBytes: 3,
},
},
},
"redis": {
MethodsStats: map[string]*SizeAndEntriesCount{
"set": {
EntriesCount: 5,
VolumeInBytes: 6,
},
},
},
},
},
}
expected := map[time.Time]map[string]map[string]*AccumulativeStatsCounter{
time.Date(2022, time.Month(1), 1, 10, 00, 00, 0, time.UTC): {
"http": {
"get": {
Name: "get",
EntriesCount: 1,
VolumeSizeBytes: 2,
},
},
"kafka": {
"listTopics": {
Name: "listTopics",
EntriesCount: 5,
VolumeSizeBytes: 6,
},
},
},
time.Date(2022, time.Month(1), 1, 10, 01, 00, 0, time.UTC): {
"http": {
"post": {
Name: "post",
EntriesCount: 2,
VolumeSizeBytes: 3,
},
"get": {
Name: "get",
EntriesCount: 1,
VolumeSizeBytes: 2,
},
},
"redis": {
"set": {
Name: "set",
EntriesCount: 5,
VolumeSizeBytes: 6,
},
},
},
}
actual := getAggregatedResultTiming(bucketStatsForTest, time.Minute)
if len(actual) != len(expected) {
t.Errorf("unexpected result - expected: %v, actual: %v", len(expected), len(actual))
}
}

84
agent/pkg/providers/stats_provider_test.go
View File

@ -1,84 +0,0 @@
package providers_test
import (
"fmt"
"testing"
"time"
"github.com/kubeshark/kubeshark/agent/pkg/providers"
"github.com/kubeshark/worker/api"
)
func TestNoEntryAddedCount(t *testing.T) {
entriesStats := providers.GetGeneralStats()
if entriesStats.EntriesCount != 0 {
t.Errorf("unexpected result - expected: %v, actual: %v", 0, entriesStats.EntriesCount)
}
if entriesStats.EntriesVolumeInGB != 0 {
t.Errorf("unexpected result - expected: %v, actual: %v", 0, entriesStats.EntriesVolumeInGB)
}
}
func TestEntryAddedCount(t *testing.T) {
tests := []int{1, 5, 10, 100, 500, 1000}
entryBucketKey := time.Date(2021, 1, 1, 10, 0, 0, 0, time.UTC)
valueLessThanBucketThreshold := time.Second * 130
mockSummery := &api.BaseEntry{Protocol: api.Protocol{ProtocolSummary: api.ProtocolSummary{Name: "mock"}}, Method: "mock-method", Timestamp: entryBucketKey.Add(valueLessThanBucketThreshold).UnixNano()}
for _, entriesCount := range tests {
t.Run(fmt.Sprintf("%d", entriesCount), func(t *testing.T) {
for i := 0; i < entriesCount; i++ {
providers.EntryAdded(0, mockSummery)
}
entriesStats := providers.GetGeneralStats()
if entriesStats.EntriesCount != entriesCount {
t.Errorf("unexpected result - expected: %v, actual: %v", entriesCount, entriesStats.EntriesCount)
}
if entriesStats.EntriesVolumeInGB != 0 {
t.Errorf("unexpected result - expected: %v, actual: %v", 0, entriesStats.EntriesVolumeInGB)
}
t.Cleanup(func() {
providers.ResetGeneralStats()
generalStats := providers.GetGeneralStats()
if generalStats.EntriesCount != 0 {
t.Errorf("unexpected result - expected: %v, actual: %v", 0, generalStats.EntriesCount)
}
})
})
}
}
func TestEntryAddedVolume(t *testing.T) {
// 6 bytes + 4 bytes
tests := [][]byte{[]byte("volume"), []byte("test")}
var expectedEntriesCount int
var expectedVolumeInGB float64
mockSummery := &api.BaseEntry{Protocol: api.Protocol{ProtocolSummary: api.ProtocolSummary{Name: "mock"}}, Method: "mock-method", Timestamp: time.Date(2021, 1, 1, 10, 0, 0, 0, time.UTC).UnixNano()}
for _, data := range tests {
t.Run(fmt.Sprintf("%d", len(data)), func(t *testing.T) {
expectedEntriesCount++
expectedVolumeInGB += float64(len(data)) / (1 << 30)
providers.EntryAdded(len(data), mockSummery)
entriesStats := providers.GetGeneralStats()
if entriesStats.EntriesCount != expectedEntriesCount {
t.Errorf("unexpected result - expected: %v, actual: %v", expectedEntriesCount, entriesStats.EntriesCount)
}
if entriesStats.EntriesVolumeInGB != expectedVolumeInGB {
t.Errorf("unexpected result - expected: %v, actual: %v", expectedVolumeInGB, entriesStats.EntriesVolumeInGB)
}
})
}
}

70
agent/pkg/providers/tappedPods/tapped_pods_provider.go
View File

@ -1,70 +0,0 @@
package tappedPods
import (
"os"
"strings"
"sync"
"github.com/kubeshark/kubeshark/agent/pkg/providers/tappers"
"github.com/kubeshark/kubeshark/agent/pkg/utils"
"github.com/kubeshark/kubeshark/logger"
"github.com/kubeshark/kubeshark/shared"
)
const FilePath = shared.DataDirPath + "tapped-pods.json"
var (
lock = &sync.Mutex{}
syncOnce sync.Once
tappedPods []*shared.PodInfo
nodeHostToTappedPodsMap shared.NodeToPodsMap
)
func Get() []*shared.PodInfo {
syncOnce.Do(func() {
if err := utils.ReadJsonFile(FilePath, &tappedPods); err != nil {
if !os.IsNotExist(err) {
logger.Log.Errorf("Error reading tapped pods from file, err: %v", err)
}
}
})
return tappedPods
}
func Set(tappedPodsToSet []*shared.PodInfo) {
lock.Lock()
defer lock.Unlock()
tappedPods = tappedPodsToSet
if err := utils.SaveJsonFile(FilePath, tappedPods); err != nil {
logger.Log.Errorf("Error saving tapped pods, err: %v", err)
}
}
func GetTappedPodsStatus() []shared.TappedPodStatus {
tappedPodsStatus := make([]shared.TappedPodStatus, 0)
tapperStatus := tappers.GetStatus()
for _, pod := range Get() {
var status string
if tapperStatus, ok := tapperStatus[pod.NodeName]; ok {
status = strings.ToLower(tapperStatus.Status)
}
isTapped := status == "running"
tappedPodsStatus = append(tappedPodsStatus, shared.TappedPodStatus{Name: pod.Name, Namespace: pod.Namespace, IsTapped: isTapped})
}
return tappedPodsStatus
}
func SetNodeToTappedPodMap(nodeToTappedPodsMap shared.NodeToPodsMap) {
summary := nodeToTappedPodsMap.Summary()
logger.Log.Debugf("Setting node to tapped pods map to %v", summary)
nodeHostToTappedPodsMap = nodeToTappedPodsMap
}
func GetNodeToTappedPodMap() shared.NodeToPodsMap {
return nodeHostToTappedPodsMap
}

83
agent/pkg/providers/tappers/tappers_provider.go
View File

@ -1,83 +0,0 @@
package tappers
import (
"os"
"sync"
"github.com/kubeshark/kubeshark/agent/pkg/utils"
"github.com/kubeshark/kubeshark/logger"
"github.com/kubeshark/kubeshark/shared"
)
const FilePath = shared.DataDirPath + "tappers-status.json"
var (
lockStatus = &sync.Mutex{}
syncOnce sync.Once
status map[string]*shared.TapperStatus
lockConnectedCount = &sync.Mutex{}
connectedCount int
)
func GetStatus() map[string]*shared.TapperStatus {
initStatus()
return status
}
func SetStatus(tapperStatus *shared.TapperStatus) {
initStatus()
lockStatus.Lock()
defer lockStatus.Unlock()
status[tapperStatus.NodeName] = tapperStatus
saveStatus()
}
func ResetStatus() {
lockStatus.Lock()
defer lockStatus.Unlock()
status = make(map[string]*shared.TapperStatus)
saveStatus()
}
func GetConnectedCount() int {
return connectedCount
}
func Connected() {
lockConnectedCount.Lock()
defer lockConnectedCount.Unlock()
connectedCount++
}
func Disconnected() {
lockConnectedCount.Lock()
defer lockConnectedCount.Unlock()
connectedCount--
}
func initStatus() {
syncOnce.Do(func() {
if err := utils.ReadJsonFile(FilePath, &status); err != nil {
status = make(map[string]*shared.TapperStatus)
if !os.IsNotExist(err) {
logger.Log.Errorf("Error reading tappers status from file, err: %v", err)
}
}
})
}
func saveStatus() {
if err := utils.SaveJsonFile(FilePath, status); err != nil {
logger.Log.Errorf("Error saving tappers status, err: %v", err)
}
}

184
agent/pkg/replay/replay.go
View File

@ -1,184 +0,0 @@
package replay
import (
"bytes"
"encoding/json"
"fmt"
"net/http"
"strings"
"sync"
"time"
"github.com/google/uuid"
"github.com/kubeshark/kubeshark/agent/pkg/app"
tapApi "github.com/kubeshark/worker/api"
kubesharkhttp "github.com/kubeshark/worker/extensions/http"
)
var (
inProcessRequestsLocker = sync.Mutex{}
inProcessRequests = 0
)
const maxParallelAction = 5
type Details struct {
Method string `json:"method"`
Url string `json:"url"`
Body string `json:"body"`
Headers map[string]string `json:"headers"`
}
type Response struct {
Success bool `json:"status"`
Data interface{} `json:"data"`
ErrorMessage string `json:"errorMessage"`
}
func incrementCounter() bool {
result := false
inProcessRequestsLocker.Lock()
if inProcessRequests < maxParallelAction {
inProcessRequests++
result = true
}
inProcessRequestsLocker.Unlock()
return result
}
func decrementCounter() {
inProcessRequestsLocker.Lock()
inProcessRequests--
inProcessRequestsLocker.Unlock()
}
func getEntryFromRequestResponse(extension *tapApi.Extension, request *http.Request, response *http.Response) *tapApi.Entry {
captureTime := time.Now()
itemTmp := tapApi.OutputChannelItem{
Protocol: *extension.Protocol,
ConnectionInfo: &tapApi.ConnectionInfo{
ClientIP: "",
ClientPort: "1",
ServerIP: "",
ServerPort: "1",
IsOutgoing: false,
},
Capture: "",
Timestamp: time.Now().UnixMilli(),
Pair: &tapApi.RequestResponsePair{
Request: tapApi.GenericMessage{
IsRequest: true,
CaptureTime: captureTime,
CaptureSize: 0,
Payload: &kubesharkhttp.HTTPPayload{
Type: kubesharkhttp.TypeHttpRequest,
Data: request,
},
},
Response: tapApi.GenericMessage{
IsRequest: false,
CaptureTime: captureTime,
CaptureSize: 0,
Payload: &kubesharkhttp.HTTPPayload{
Type: kubesharkhttp.TypeHttpResponse,
Data: response,
},
},
},
}
// Analyze is expecting an item that's marshalled and unmarshalled
itemMarshalled, err := json.Marshal(itemTmp)
if err != nil {
return nil
}
var finalItem *tapApi.OutputChannelItem
if err := json.Unmarshal(itemMarshalled, &finalItem); err != nil {
return nil
}
return extension.Dissector.Analyze(finalItem, "", "", "")
}
func ExecuteRequest(replayData *Details, timeout time.Duration) *Response {
if incrementCounter() {
defer decrementCounter()
client := &http.Client{
Timeout: timeout,
}
request, err := http.NewRequest(strings.ToUpper(replayData.Method), replayData.Url, bytes.NewBufferString(replayData.Body))
if err != nil {
return &Response{
Success: false,
Data: nil,
ErrorMessage: err.Error(),
}
}
for headerKey, headerValue := range replayData.Headers {
request.Header.Add(headerKey, headerValue)
}
request.Header.Add("x-kubeshark", uuid.New().String())
response, requestErr := client.Do(request)
if requestErr != nil {
return &Response{
Success: false,
Data: nil,
ErrorMessage: requestErr.Error(),
}
}
extension := app.ExtensionsMap["http"] // # TODO: maybe pass the extension to the function so it can be tested
entry := getEntryFromRequestResponse(extension, request, response)
base := extension.Dissector.Summarize(entry)
var representation []byte
// Represent is expecting an entry that's marshalled and unmarshalled
entryMarshalled, err := json.Marshal(entry)
if err != nil {
return &Response{
Success: false,
Data: nil,
ErrorMessage: err.Error(),
}
}
var entryUnmarshalled *tapApi.Entry
if err := json.Unmarshal(entryMarshalled, &entryUnmarshalled); err != nil {
return &Response{
Success: false,
Data: nil,
ErrorMessage: err.Error(),
}
}
representation, err = extension.Dissector.Represent(entryUnmarshalled.Request, entryUnmarshalled.Response)
if err != nil {
return &Response{
Success: false,
Data: nil,
ErrorMessage: err.Error(),
}
}
return &Response{
Success: true,
Data: &tapApi.EntryWrapper{
Protocol: *extension.Protocol,
Representation: string(representation),
Data: entryUnmarshalled,
Base: base,
},
ErrorMessage: "",
}
} else {
return &Response{
Success: false,
Data: nil,
ErrorMessage: fmt.Sprintf("reached threshold of %d requests", maxParallelAction),
}
}
}

106
agent/pkg/replay/replay_internal_test.go
View File

@ -1,106 +0,0 @@
package replay
import (
"bytes"
"fmt"
"net/http"
"strings"
"testing"
"time"
"encoding/json"
"github.com/google/uuid"
tapApi "github.com/kubeshark/worker/api"
kubesharkhttp "github.com/kubeshark/worker/extensions/http"
)
func TestValid(t *testing.T) {
client := &http.Client{
Timeout: 10 * time.Second,
}
tests := map[string]*Details{
"40x": {
Method: "GET",
Url: "http://httpbin.org/status/404",
Body: "",
Headers: map[string]string{},
},
"20x": {
Method: "GET",
Url: "http://httpbin.org/status/200",
Body: "",
Headers: map[string]string{},
},
"50x": {
Method: "GET",
Url: "http://httpbin.org/status/500",
Body: "",
Headers: map[string]string{},
},
// TODO: this should be fixes, currently not working because of header name with ":"
//":path-header": {
// Method: "GET",
// Url: "http://httpbin.org/get",
// Body: "",
// Headers: map[string]string{
// ":path": "/get",
// },
// },
}
for testCaseName, replayData := range tests {
t.Run(fmt.Sprintf("%+v", testCaseName), func(t *testing.T) {
request, err := http.NewRequest(strings.ToUpper(replayData.Method), replayData.Url, bytes.NewBufferString(replayData.Body))
if err != nil {
t.Errorf("Error executing request")
}
for headerKey, headerValue := range replayData.Headers {
request.Header.Add(headerKey, headerValue)
}
request.Header.Add("x-kubeshark", uuid.New().String())
response, requestErr := client.Do(request)
if requestErr != nil {
t.Errorf("failed: %v, ", requestErr)
}
extensionHttp := &tapApi.Extension{}
dissectorHttp := kubesharkhttp.NewDissector()
dissectorHttp.Register(extensionHttp)
extensionHttp.Dissector = dissectorHttp
extension := extensionHttp
entry := getEntryFromRequestResponse(extension, request, response)
base := extension.Dissector.Summarize(entry)
// Represent is expecting an entry that's marshalled and unmarshalled
entryMarshalled, err := json.Marshal(entry)
if err != nil {
t.Errorf("failed marshaling entry: %v, ", err)
}
var entryUnmarshalled *tapApi.Entry
if err := json.Unmarshal(entryMarshalled, &entryUnmarshalled); err != nil {
t.Errorf("failed unmarshaling entry: %v, ", err)
}
var representation []byte
representation, err = extension.Dissector.Represent(entryUnmarshalled.Request, entryUnmarshalled.Response)
if err != nil {
t.Errorf("failed: %v, ", err)
}
result := &tapApi.EntryWrapper{
Protocol: *extension.Protocol,
Representation: string(representation),
Data: entry,
Base: base,
}
t.Logf("%+v", result)
//data, _ := json.MarshalIndent(result, "", " ")
//t.Logf("%+v", string(data))
})
}
}

69
agent/pkg/resolver/README.md
View File

@ -1,69 +0,0 @@
<!--
(NOT RELEVANT CURRENTLY)
## Installation
To be able to import this package, you must add `replace github.com/kubeshark/kubeshark/resolver => ../resolver` to the end of your `go.mod` file
And then add `github.com/kubeshark/kubeshark/resolver v0.0.0` to your require block
full example `go.mod`:
```
module github.com/kubeshark/kubeshark/cli
go 1.16
require (
github.com/spf13/cobra v1.1.3
github.com/kubeshark/kubeshark/resolver v0.0.0
k8s.io/api v0.21.0
k8s.io/apimachinery v0.21.0
k8s.io/client-go v0.21.0
)
replace github.com/kubeshark/kubeshark/resolver => ../resolver
```
Now you will be able to import `github.com/kubeshark/kubeshark/resolver` in any `.go` file
-->
## Usage
### Full example
``` go
errOut := make(chan error, 100)
k8sResolver, err := resolver.NewFromOutOfCluster("", errOut)
if err != nil {
logger.Log.Errorf("error creating k8s resolver %s", err)
}
ctx, cancel := context.WithCancel(context.Background())
k8sResolver.Start(ctx)
resolvedName := k8sResolver.Resolve("10.107.251.91") // will always return `nil` in real scenarios as the internal map takes a moment to populate after `Start` is called
if resolvedName != nil {
logger.Log.Errorf("resolved 10.107.251.91=%s", *resolvedName)
} else {
logger.Log.Error("Could not find a resolved name for 10.107.251.91")
}
for {
select {
case err := <- errOut:
logger.Log.Errorf("name resolving error %s", err)
}
}
```
### In cluster authentication
Create resolver using the function `NewFromInCluster(errOut chan error)`
### Out of cluster authentication
Create resolver using the function `NewFromOutOfCluster(kubeConfigPath string, errOut chan error)`
the `kubeConfigPath` param is optional, pass an empty string `""` for resolver to auto locate the default kubeconfig file
### Error handling
Please ensure there is always a thread reading from the `errOut` channel, not doing so will result in the resolver threads getting blocked and the resolver will fail to update.
Also note that any error you receive through this channel does not necessarily mean that resolver is no longer running. the resolver will infinitely retry watching k8s resources until the provided context is cancelled.

23
agent/pkg/resolver/loader.go
View File

@ -1,23 +0,0 @@
package resolver
import (
cmap "github.com/orcaman/concurrent-map"
"k8s.io/client-go/kubernetes"
_ "k8s.io/client-go/plugin/pkg/client/auth/azure"
_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
_ "k8s.io/client-go/plugin/pkg/client/auth/oidc"
_ "k8s.io/client-go/plugin/pkg/client/auth/openstack"
restclient "k8s.io/client-go/rest"
)
func NewFromInCluster(errOut chan error, namespace string) (*Resolver, error) {
config, err := restclient.InClusterConfig()
if err != nil {
return nil, err
}
clientSet, err := kubernetes.NewForConfig(config)
if err != nil {
return nil, err
}
return &Resolver{clientConfig: config, clientSet: clientSet, nameMap: cmap.New(), serviceMap: cmap.New(), errOut: errOut, namespace: namespace}, nil
}

208
agent/pkg/resolver/resolver.go
View File

@ -1,208 +0,0 @@
package resolver
import (
"context"
"errors"
"fmt"
"github.com/kubeshark/kubeshark/logger"
k8serrors "k8s.io/apimachinery/pkg/api/errors"
cmap "github.com/orcaman/concurrent-map"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/watch"
"k8s.io/client-go/kubernetes"
restclient "k8s.io/client-go/rest"
)
const (
kubClientNullString = "None"
)
type Resolver struct {
clientConfig *restclient.Config
clientSet *kubernetes.Clientset
nameMap cmap.ConcurrentMap
serviceMap cmap.ConcurrentMap
isStarted bool
errOut chan error
namespace string
}
type ResolvedObjectInfo struct {
FullAddress string
Namespace string
}
func (resolver *Resolver) Start(ctx context.Context) {
if !resolver.isStarted {
resolver.isStarted = true
go resolver.infiniteErrorHandleRetryFunc(ctx, resolver.watchServices)
go resolver.infiniteErrorHandleRetryFunc(ctx, resolver.watchEndpoints)
go resolver.infiniteErrorHandleRetryFunc(ctx, resolver.watchPods)
}
}
func (resolver *Resolver) Resolve(name string) *ResolvedObjectInfo {
resolvedName, isFound := resolver.nameMap.Get(name)
if !isFound {
return nil
}
return resolvedName.(*ResolvedObjectInfo)
}
func (resolver *Resolver) GetMap() cmap.ConcurrentMap {
return resolver.nameMap
}
func (resolver *Resolver) CheckIsServiceIP(address string) bool {
_, isFound := resolver.serviceMap.Get(address)
return isFound
}
func (resolver *Resolver) watchPods(ctx context.Context) error {
// empty namespace makes the client watch all namespaces
watcher, err := resolver.clientSet.CoreV1().Pods(resolver.namespace).Watch(ctx, metav1.ListOptions{Watch: true})
if err != nil {
return err
}
for {
select {
case event := <-watcher.ResultChan():
if event.Object == nil {
return errors.New("error in kubectl pod watch")
}
if event.Type == watch.Deleted {
pod := event.Object.(*corev1.Pod)
resolver.saveResolvedName(pod.Status.PodIP, "", pod.Namespace, event.Type)
}
case <-ctx.Done():
watcher.Stop()
return nil
}
}
}
func (resolver *Resolver) watchEndpoints(ctx context.Context) error {
// empty namespace makes the client watch all namespaces
watcher, err := resolver.clientSet.CoreV1().Endpoints(resolver.namespace).Watch(ctx, metav1.ListOptions{Watch: true})
if err != nil {
return err
}
for {
select {
case event := <-watcher.ResultChan():
if event.Object == nil {
return errors.New("error in kubectl endpoint watch")
}
endpoint := event.Object.(*corev1.Endpoints)
serviceHostname := fmt.Sprintf("%s.%s", endpoint.Name, endpoint.Namespace)
if endpoint.Subsets != nil {
for _, subset := range endpoint.Subsets {
var ports []int32
if subset.Ports != nil {
for _, portMapping := range subset.Ports {
if portMapping.Port > 0 {
ports = append(ports, portMapping.Port)
}
}
}
if subset.Addresses != nil {
for _, address := range subset.Addresses {
resolver.saveResolvedName(address.IP, serviceHostname, endpoint.Namespace, event.Type)
for _, port := range ports {
ipWithPort := fmt.Sprintf("%s:%d", address.IP, port)
resolver.saveResolvedName(ipWithPort, serviceHostname, endpoint.Namespace, event.Type)
}
}
}
}
}
case <-ctx.Done():
watcher.Stop()
return nil
}
}
}
func (resolver *Resolver) watchServices(ctx context.Context) error {
// empty namespace makes the client watch all namespaces
watcher, err := resolver.clientSet.CoreV1().Services(resolver.namespace).Watch(ctx, metav1.ListOptions{Watch: true})
if err != nil {
return err
}
for {
select {
case event := <-watcher.ResultChan():
if event.Object == nil {
return errors.New("error in kubectl service watch")
}
service := event.Object.(*corev1.Service)
serviceHostname := fmt.Sprintf("%s.%s", service.Name, service.Namespace)
if service.Spec.ClusterIP != "" && service.Spec.ClusterIP != kubClientNullString {
resolver.saveResolvedName(service.Spec.ClusterIP, serviceHostname, service.Namespace, event.Type)
if service.Spec.Ports != nil {
for _, port := range service.Spec.Ports {
if port.Port > 0 {
resolver.saveResolvedName(fmt.Sprintf("%s:%d", service.Spec.ClusterIP, port.Port), serviceHostname, service.Namespace, event.Type)
}
}
}
resolver.saveServiceIP(service.Spec.ClusterIP, serviceHostname, service.Namespace, event.Type)
}
if service.Status.LoadBalancer.Ingress != nil {
for _, ingress := range service.Status.LoadBalancer.Ingress {
resolver.saveResolvedName(ingress.IP, serviceHostname, service.Namespace, event.Type)
}
}
case <-ctx.Done():
watcher.Stop()
return nil
}
}
}
func (resolver *Resolver) saveResolvedName(key string, resolved string, namespace string, eventType watch.EventType) {
if eventType == watch.Deleted {
resolver.nameMap.Remove(resolved)
resolver.nameMap.Remove(key)
logger.Log.Infof("setting %s=nil", key)
} else {
resolver.nameMap.Set(key, &ResolvedObjectInfo{FullAddress: resolved, Namespace: namespace})
resolver.nameMap.Set(resolved, &ResolvedObjectInfo{FullAddress: resolved, Namespace: namespace})
logger.Log.Infof("setting %s=%s", key, resolved)
}
}
func (resolver *Resolver) saveServiceIP(key string, resolved string, namespace string, eventType watch.EventType) {
if eventType == watch.Deleted {
resolver.serviceMap.Remove(key)
} else {
resolver.nameMap.Set(key, &ResolvedObjectInfo{FullAddress: resolved, Namespace: namespace})
}
}
func (resolver *Resolver) infiniteErrorHandleRetryFunc(ctx context.Context, fun func(ctx context.Context) error) {
for {
err := fun(ctx)
if err != nil {
resolver.errOut <- err
var statusError *k8serrors.StatusError
if errors.As(err, &statusError) {
if statusError.ErrStatus.Reason == metav1.StatusReasonForbidden {
logger.Log.Infof("Resolver loop encountered permission error, aborting event listening - %v", err)
return
}
}
}
if ctx.Err() != nil { // context was cancelled or errored
return
}
}
}

15
agent/pkg/routes/db_routes.go
View File

@ -1,15 +0,0 @@
package routes
import (
"github.com/kubeshark/kubeshark/agent/pkg/controllers"
"github.com/gin-gonic/gin"
)
// DdRoutes defines the group of database routes.
func DbRoutes(app *gin.Engine) {
routeGroup := app.Group("/db")
routeGroup.GET("/flush", controllers.Flush)
routeGroup.GET("/reset", controllers.Reset)
}

14
agent/pkg/routes/entries_routes.go
View File

@ -1,14 +0,0 @@
package routes
import (
"github.com/gin-gonic/gin"
"github.com/kubeshark/kubeshark/agent/pkg/controllers"
)
// EntriesRoutes defines the group of har entries routes.
func EntriesRoutes(ginApp *gin.Engine) {
routeGroup := ginApp.Group("/entries")
routeGroup.GET("/", controllers.GetEntries) // get entries (base/thin entries) and metadata
routeGroup.GET("/:id", controllers.GetEntry) // get single (full) entry
}

14
agent/pkg/routes/metadata_routes.go
View File

@ -1,14 +0,0 @@
package routes
import (
"github.com/kubeshark/kubeshark/agent/pkg/controllers"
"github.com/gin-gonic/gin"
)
// MetadataRoutes defines the group of metadata routes.
func MetadataRoutes(app *gin.Engine) {
routeGroup := app.Group("/metadata")
routeGroup.GET("/version", controllers.GetVersion)
}

15
agent/pkg/routes/oas_routes.go
View File

@ -1,15 +0,0 @@
package routes
import (
"github.com/gin-gonic/gin"
"github.com/kubeshark/kubeshark/agent/pkg/controllers"
)
// OASRoutes methods to access OAS spec
func OASRoutes(ginApp *gin.Engine) {
routeGroup := ginApp.Group("/oas")
routeGroup.GET("/", controllers.GetOASServers) // list of servers in OAS map
routeGroup.GET("/all", controllers.GetOASAllSpecs) // list of servers in OAS map
routeGroup.GET("/:id", controllers.GetOASSpec) // get OAS spec for given server
}

12
agent/pkg/routes/query_routes.go
View File

@ -1,12 +0,0 @@
package routes
import (
"github.com/gin-gonic/gin"
"github.com/kubeshark/kubeshark/agent/pkg/controllers"
)
func QueryRoutes(ginApp *gin.Engine) {
routeGroup := ginApp.Group("/query")
routeGroup.POST("/validate", controllers.PostValidate)
}

13
agent/pkg/routes/replay_routes.go
View File

@ -1,13 +0,0 @@
package routes
import (
"github.com/gin-gonic/gin"
"github.com/kubeshark/kubeshark/agent/pkg/controllers"
)
// ReplayRoutes defines the group of replay routes.
func ReplayRoutes(app *gin.Engine) {
routeGroup := app.Group("/replay")
routeGroup.POST("/", controllers.ReplayRequest)
}

16
agent/pkg/routes/service_map_routes.go
View File

@ -1,16 +0,0 @@
package routes
import (
"github.com/gin-gonic/gin"
"github.com/kubeshark/kubeshark/agent/pkg/controllers"
)
func ServiceMapRoutes(ginApp *gin.Engine) {
routeGroup := ginApp.Group("/servicemap")
controller := controllers.NewServiceMapController()
routeGroup.GET("/status", controller.Status)
routeGroup.GET("/get", controller.Get)
routeGroup.GET("/reset", controller.Reset)
}

22
agent/pkg/routes/status_routes.go
View File

@ -1,22 +0,0 @@
package routes
import (
"github.com/gin-gonic/gin"
"github.com/kubeshark/kubeshark/agent/pkg/controllers"
)
func StatusRoutes(ginApp *gin.Engine) {
routeGroup := ginApp.Group("/status")
routeGroup.GET("/health", controllers.HealthCheck)
routeGroup.POST("/tappedPods", controllers.PostTappedPods)
routeGroup.POST("/tapperStatus", controllers.PostTapperStatus)
routeGroup.GET("/connectedTappersCount", controllers.GetConnectedTappersCount)
routeGroup.GET("/tap", controllers.GetTappingStatus)
routeGroup.GET("/general", controllers.GetGeneralStats)
routeGroup.GET("/trafficStats", controllers.GetTrafficStats)
routeGroup.GET("/resolving", controllers.GetCurrentResolvingInformation)
}

33
agent/pkg/servicemap/models.go
View File

@ -1,33 +0,0 @@
package servicemap
import (
tapApi "github.com/kubeshark/worker/api"
)
type ServiceMapStatus struct {
Status string `json:"status"`
EntriesProcessedCount int `json:"entriesProcessedCount"`
NodeCount int `json:"nodeCount"`
EdgeCount int `json:"edgeCount"`
}
type ServiceMapResponse struct {
Status ServiceMapStatus `json:"status"`
Nodes []ServiceMapNode `json:"nodes"`
Edges []ServiceMapEdge `json:"edges"`
}
type ServiceMapNode struct {
Id int `json:"id"`
Name string `json:"name"`
Entry *tapApi.TCP `json:"entry"`
Count int `json:"count"`
Resolved bool `json:"resolved"`
}
type ServiceMapEdge struct {
Source ServiceMapNode `json:"source"`
Destination ServiceMapNode `json:"destination"`
Count int `json:"count"`
Protocol *tapApi.Protocol `json:"protocol"`
}

306
agent/pkg/servicemap/servicemap.go
View File

@ -1,306 +0,0 @@
package servicemap
import (
"sync"
"github.com/jinzhu/copier"
"github.com/kubeshark/kubeshark/logger"
tapApi "github.com/kubeshark/worker/api"
)
const (
ServiceMapEnabled = "enabled"
ServiceMapDisabled = "disabled"
UnresolvedNodeName = "unresolved"
)
var instance *defaultServiceMap
var once sync.Once
func GetDefaultServiceMapInstance() *defaultServiceMap {
once.Do(func() {
instance = NewDefaultServiceMapGenerator()
logger.Log.Debug("Service Map Initialized")
})
return instance
}
type defaultServiceMap struct {
enabled bool
graph *graph
entriesProcessed int
}
type ServiceMapSink interface {
NewTCPEntry(source *tapApi.TCP, destination *tapApi.TCP, protocol *tapApi.Protocol)
}
type ServiceMap interface {
Enable()
Disable()
IsEnabled() bool
GetStatus() ServiceMapStatus
GetNodes() []ServiceMapNode
GetEdges() []ServiceMapEdge
GetEntriesProcessedCount() int
GetNodesCount() int
GetEdgesCount() int
Reset()
}
func NewDefaultServiceMapGenerator() *defaultServiceMap {
return &defaultServiceMap{
enabled: false,
entriesProcessed: 0,
graph: newDirectedGraph(),
}
}
type key string
type entryData struct {
key key
entry *tapApi.TCP
}
type nodeData struct {
id int
entry *tapApi.TCP
count int
}
type edgeProtocol struct {
protocol *tapApi.Protocol
count int
}
type edgeData struct {
data map[key]*edgeProtocol
}
type graph struct {
Nodes map[key]*nodeData
Edges map[key]map[key]*edgeData
}
func newDirectedGraph() *graph {
return &graph{
Nodes: make(map[key]*nodeData),
Edges: make(map[key]map[key]*edgeData),
}
}
func newNodeData(id int, e *tapApi.TCP) *nodeData {
return &nodeData{
id: id,
entry: e,
count: 1,
}
}
func newEdgeData(p *tapApi.Protocol) *edgeData {
return &edgeData{
data: map[key]*edgeProtocol{
key(p.Name): {
protocol: p,
count: 1,
},
},
}
}
func (s *defaultServiceMap) nodeExists(k key) (*nodeData, bool) {
n, ok := s.graph.Nodes[k]
return n, ok
}
func (s *defaultServiceMap) addNode(k key, e *tapApi.TCP) (*nodeData, bool) {
nd, exists := s.nodeExists(k)
if !exists {
s.graph.Nodes[k] = newNodeData(len(s.graph.Nodes)+1, e)
return s.graph.Nodes[k], true
}
return nd, false
}
func (s *defaultServiceMap) addEdge(u, v *entryData, p *tapApi.Protocol) {
if n, ok := s.addNode(u.key, u.entry); !ok {
n.count++
}
if n, ok := s.addNode(v.key, v.entry); !ok {
n.count++
}
if _, ok := s.graph.Edges[u.key]; !ok {
s.graph.Edges[u.key] = make(map[key]*edgeData)
}
// new edge u -> v pair
// protocol is the same for u and v
if e, ok := s.graph.Edges[u.key][v.key]; ok {
// edge data already exists for u -> v pair
// we have a new protocol for this u -> v pair
k := key(p.Name)
if pd, pOk := e.data[k]; pOk {
// protocol key already exists, just increment the count
pd.count++
} else {
// new protocol key
e.data[k] = &edgeProtocol{
protocol: p,
count: 1,
}
}
} else {
// new edge data for u -> v pair
s.graph.Edges[u.key][v.key] = newEdgeData(p)
}
s.entriesProcessed++
}
func (s *defaultServiceMap) Enable() {
s.enabled = true
}
func (s *defaultServiceMap) Disable() {
s.Reset()
s.enabled = false
}
func (s *defaultServiceMap) IsEnabled() bool {
return s.enabled
}
func (s *defaultServiceMap) NewTCPEntry(src *tapApi.TCP, dst *tapApi.TCP, p *tapApi.Protocol) {
if !s.IsEnabled() {
return
}
var srcEntry *entryData
var dstEntry *entryData
if len(src.Name) == 0 {
srcEntry = &entryData{
key: key(src.IP),
entry: &tapApi.TCP{},
}
if err := copier.Copy(srcEntry.entry, src); err != nil {
logger.Log.Errorf("Error while copying src entry into src entry data")
}
srcEntry.entry.Name = UnresolvedNodeName
} else {
srcEntry = &entryData{
key: key(src.Name),
entry: src,
}
}
if len(dst.Name) == 0 {
dstEntry = &entryData{
key: key(dst.IP),
entry: &tapApi.TCP{},
}
if err := copier.Copy(dstEntry.entry, dst); err != nil {
logger.Log.Errorf("Error while copying dst entry into dst entry data")
}
dstEntry.entry.Name = UnresolvedNodeName
} else {
dstEntry = &entryData{
key: key(dst.Name),
entry: dst,
}
}
s.addEdge(srcEntry, dstEntry, p)
}
func (s *defaultServiceMap) GetStatus() ServiceMapStatus {
status := ServiceMapDisabled
if s.IsEnabled() {
status = ServiceMapEnabled
}
return ServiceMapStatus{
Status: status,
EntriesProcessedCount: s.entriesProcessed,
NodeCount: s.GetNodesCount(),
EdgeCount: s.GetEdgesCount(),
}
}
func (s *defaultServiceMap) GetNodes() []ServiceMapNode {
nodes := []ServiceMapNode{}
for i, n := range s.graph.Nodes {
nodes = append(nodes, ServiceMapNode{
Id: n.id,
Name: string(i),
Resolved: n.entry.Name != UnresolvedNodeName,
Entry: n.entry,
Count: n.count,
})
}
return nodes
}
func (s *defaultServiceMap) GetEdges() []ServiceMapEdge {
edges := []ServiceMapEdge{}
for u, m := range s.graph.Edges {
for v := range m {
for _, p := range s.graph.Edges[u][v].data {
edges = append(edges, ServiceMapEdge{
Source: ServiceMapNode{
Id: s.graph.Nodes[u].id,
Name: string(u),
Entry: s.graph.Nodes[u].entry,
Resolved: s.graph.Nodes[u].entry.Name != UnresolvedNodeName,
Count: s.graph.Nodes[u].count,
},
Destination: ServiceMapNode{
Id: s.graph.Nodes[v].id,
Name: string(v),
Entry: s.graph.Nodes[v].entry,
Resolved: s.graph.Nodes[v].entry.Name != UnresolvedNodeName,
Count: s.graph.Nodes[v].count,
},
Count: p.count,
Protocol: p.protocol,
})
}
}
}
return edges
}
func (s *defaultServiceMap) GetEntriesProcessedCount() int {
return s.entriesProcessed
}
func (s *defaultServiceMap) GetNodesCount() int {
return len(s.graph.Nodes)
}
func (s *defaultServiceMap) GetEdgesCount() int {
var count int
for u, m := range s.graph.Edges {
for v := range m {
for range s.graph.Edges[u][v].data {
count++
}
}
}
return count
}
func (s *defaultServiceMap) Reset() {
s.entriesProcessed = 0
s.graph = newDirectedGraph()
}

419
agent/pkg/servicemap/servicemap_test.go
View File

@ -1,419 +0,0 @@
package servicemap
import (
"fmt"
"strings"
"testing"
tapApi "github.com/kubeshark/worker/api"
"github.com/stretchr/testify/suite"
)
const (
a = "aService"
b = "bService"
c = "cService"
d = "dService"
Ip = "127.0.0.1"
Port = "80"
)
var (
TCPEntryA = &tapApi.TCP{
Name: a,
Port: Port,
IP: fmt.Sprintf("%s.%s", Ip, a),
}
TCPEntryB = &tapApi.TCP{
Name: b,
Port: Port,
IP: fmt.Sprintf("%s.%s", Ip, b),
}
TCPEntryC = &tapApi.TCP{
Name: c,
Port: Port,
IP: fmt.Sprintf("%s.%s", Ip, c),
}
TCPEntryD = &tapApi.TCP{
Name: d,
Port: Port,
IP: fmt.Sprintf("%s.%s", Ip, d),
}
TCPEntryUnresolved = &tapApi.TCP{
Name: "",
Port: Port,
IP: Ip,
}
TCPEntryUnresolved2 = &tapApi.TCP{
Name: "",
Port: Port,
IP: fmt.Sprintf("%s.%s", Ip, UnresolvedNodeName),
}
ProtocolHttp = &tapApi.Protocol{
ProtocolSummary: tapApi.ProtocolSummary{
Name: "http",
Version: "1.1",
Abbreviation: "HTTP",
},
LongName: "Hypertext Transfer Protocol -- HTTP/1.1",
Macro: "http",
BackgroundColor: "#326de6",
ForegroundColor: "#ffffff",
FontSize: 12,
ReferenceLink: "https://datatracker.ietf.org/doc/html/rfc2616",
Ports: []string{"80", "443", "8080"},
Priority: 0,
}
ProtocolRedis = &tapApi.Protocol{
ProtocolSummary: tapApi.ProtocolSummary{
Name: "redis",
Version: "3.x",
Abbreviation: "REDIS",
},
LongName: "Redis Serialization Protocol",
Macro: "redis",
BackgroundColor: "#a41e11",
ForegroundColor: "#ffffff",
FontSize: 11,
ReferenceLink: "https://redis.io/topics/protocol",
Ports: []string{"6379"},
Priority: 3,
}
)
type ServiceMapDisabledSuite struct {
suite.Suite
instance *defaultServiceMap
}
type ServiceMapEnabledSuite struct {
suite.Suite
instance *defaultServiceMap
}
func (s *ServiceMapDisabledSuite) SetupTest() {
s.instance = GetDefaultServiceMapInstance()
}
func (s *ServiceMapEnabledSuite) SetupTest() {
s.instance = GetDefaultServiceMapInstance()
s.instance.Enable()
}
func (s *ServiceMapDisabledSuite) TestServiceMapInstance() {
assert := s.Assert()
assert.NotNil(s.instance)
}
func (s *ServiceMapDisabledSuite) TestServiceMapSingletonInstance() {
assert := s.Assert()
instance2 := GetDefaultServiceMapInstance()
assert.NotNil(s.instance)
assert.NotNil(instance2)
assert.Equal(s.instance, instance2)
}
func (s *ServiceMapDisabledSuite) TestServiceMapIsEnabledShouldReturnFalseByDefault() {
assert := s.Assert()
enabled := s.instance.IsEnabled()
assert.False(enabled)
}
func (s *ServiceMapDisabledSuite) TestGetStatusShouldReturnDisabledByDefault() {
assert := s.Assert()
status := s.instance.GetStatus()
assert.Equal("disabled", status.Status)
assert.Equal(0, status.EntriesProcessedCount)
assert.Equal(0, status.NodeCount)
assert.Equal(0, status.EdgeCount)
}
func (s *ServiceMapDisabledSuite) TestNewTCPEntryShouldDoNothingWhenDisabled() {
assert := s.Assert()
s.instance.NewTCPEntry(TCPEntryA, TCPEntryB, ProtocolHttp)
s.instance.NewTCPEntry(TCPEntryC, TCPEntryD, ProtocolHttp)
status := s.instance.GetStatus()
assert.Equal("disabled", status.Status)
assert.Equal(0, status.EntriesProcessedCount)
assert.Equal(0, status.NodeCount)
assert.Equal(0, status.EdgeCount)
}
// Enabled
func (s *ServiceMapEnabledSuite) TestServiceMapIsEnabled() {
assert := s.Assert()
enabled := s.instance.IsEnabled()
assert.True(enabled)
}
func (s *ServiceMapEnabledSuite) TestServiceMap() {
assert := s.Assert()
// A -> B - HTTP
s.instance.NewTCPEntry(TCPEntryA, TCPEntryB, ProtocolHttp)
nodes := s.instance.GetNodes()
edges := s.instance.GetEdges()
// Counts for the first entry
assert.Equal(1, s.instance.GetEntriesProcessedCount())
assert.Equal(2, s.instance.GetNodesCount())
assert.Equal(2, len(nodes))
assert.Equal(1, s.instance.GetEdgesCount())
assert.Equal(1, len(edges))
//http protocol
assert.Equal(1, edges[0].Count)
assert.Equal(ProtocolHttp.Name, edges[0].Protocol.Name)
// same A -> B - HTTP, http protocol count should be 2, edges count should be 1
s.instance.NewTCPEntry(TCPEntryA, TCPEntryB, ProtocolHttp)
nodes = s.instance.GetNodes()
edges = s.instance.GetEdges()
// Counts for a second entry
assert.Equal(2, s.instance.GetEntriesProcessedCount())
assert.Equal(2, s.instance.GetNodesCount())
assert.Equal(2, len(nodes))
// edges count should still be 1, but http protocol count should be 2
assert.Equal(1, s.instance.GetEdgesCount())
assert.Equal(1, len(edges))
// http protocol
assert.Equal(2, edges[0].Count) //http
assert.Equal(ProtocolHttp.Name, edges[0].Protocol.Name)
// same A -> B - REDIS, http protocol count should be 2 and redis protocol count should 1, edges count should be 2
s.instance.NewTCPEntry(TCPEntryA, TCPEntryB, ProtocolRedis)
nodes = s.instance.GetNodes()
edges = s.instance.GetEdges()
// Counts after second entry
assert.Equal(3, s.instance.GetEntriesProcessedCount())
assert.Equal(2, s.instance.GetNodesCount())
assert.Equal(2, len(nodes))
// edges count should be 2, http protocol count should be 2 and redis protocol should be 1
assert.Equal(2, s.instance.GetEdgesCount())
assert.Equal(2, len(edges))
// http and redis protocols
httpIndex := -1
redisIndex := -1
for i, e := range edges {
if e.Protocol.Name == ProtocolHttp.Name {
httpIndex = i
continue
}
if e.Protocol.Name == ProtocolRedis.Name {
redisIndex = i
}
}
assert.NotEqual(-1, httpIndex)
assert.NotEqual(-1, redisIndex)
// http protocol
assert.Equal(2, edges[httpIndex].Count)
assert.Equal(ProtocolHttp.Name, edges[httpIndex].Protocol.Name)
// redis protocol
assert.Equal(1, edges[redisIndex].Count)
assert.Equal(ProtocolRedis.Name, edges[redisIndex].Protocol.Name)
// other entries
s.instance.NewTCPEntry(TCPEntryUnresolved, TCPEntryA, ProtocolHttp)
s.instance.NewTCPEntry(TCPEntryB, TCPEntryUnresolved2, ProtocolHttp)
s.instance.NewTCPEntry(TCPEntryC, TCPEntryD, ProtocolHttp)
s.instance.NewTCPEntry(TCPEntryA, TCPEntryC, ProtocolHttp)
status := s.instance.GetStatus()
nodes = s.instance.GetNodes()
edges = s.instance.GetEdges()
expectedEntriesProcessedCount := 7
expectedNodeCount := 6
expectedEdgeCount := 6
// Counts after all entries
assert.Equal(expectedEntriesProcessedCount, s.instance.GetEntriesProcessedCount())
assert.Equal(expectedNodeCount, s.instance.GetNodesCount())
assert.Equal(expectedNodeCount, len(nodes))
assert.Equal(expectedEdgeCount, s.instance.GetEdgesCount())
assert.Equal(expectedEdgeCount, len(edges))
// Status
assert.Equal("enabled", status.Status)
assert.Equal(expectedEntriesProcessedCount, status.EntriesProcessedCount)
assert.Equal(expectedNodeCount, status.NodeCount)
assert.Equal(expectedEdgeCount, status.EdgeCount)
// Nodes
aNode := -1
bNode := -1
cNode := -1
dNode := -1
unresolvedNode := -1
unresolvedNode2 := -1
var validateNode = func(node ServiceMapNode, entryName string, count int) int {
// id
assert.GreaterOrEqual(node.Id, 1)
assert.LessOrEqual(node.Id, expectedNodeCount)
// entry
// node.Name is the key of the node, key = entry.Name by default
// entry.Name is the name of the service and could be unresolved
// when entry.Name is unresolved, key = entry.IP
if node.Entry.Name == UnresolvedNodeName {
assert.Equal(node.Name, node.Entry.IP)
} else {
assert.Equal(node.Name, node.Entry.Name)
}
assert.Equal(Port, node.Entry.Port)
assert.Equal(entryName, node.Entry.Name)
// count
assert.Equal(count, node.Count)
return node.Id
}
for _, v := range nodes {
if strings.HasSuffix(v.Name, a) {
aNode = validateNode(v, a, 5)
continue
}
if strings.HasSuffix(v.Name, b) {
bNode = validateNode(v, b, 4)
continue
}
if strings.HasSuffix(v.Name, c) {
cNode = validateNode(v, c, 2)
continue
}
if strings.HasSuffix(v.Name, d) {
dNode = validateNode(v, d, 1)
continue
}
if v.Name == Ip {
unresolvedNode = validateNode(v, UnresolvedNodeName, 1)
continue
}
if strings.HasSuffix(v.Name, UnresolvedNodeName) {
unresolvedNode2 = validateNode(v, UnresolvedNodeName, 1)
continue
}
}
// Make sure we found all the nodes
nodeIds := [...]int{aNode, bNode, cNode, dNode, unresolvedNode, unresolvedNode2}
for _, v := range nodeIds {
assert.NotEqual(-1, v)
}
// Edges
abEdge := -1
uaEdge := -1
buEdge := -1
cdEdge := -1
acEdge := -1
var validateEdge = func(edge ServiceMapEdge, sourceEntryName string, destEntryName string, protocolName string, protocolCount int) {
// source node
assert.Contains(nodeIds, edge.Source.Id)
assert.LessOrEqual(edge.Source.Id, expectedNodeCount)
if edge.Source.Entry.Name == UnresolvedNodeName {
assert.Equal(edge.Source.Name, edge.Source.Entry.IP)
} else {
assert.Equal(edge.Source.Name, edge.Source.Entry.Name)
}
assert.Equal(sourceEntryName, edge.Source.Entry.Name)
// destination node
assert.Contains(nodeIds, edge.Destination.Id)
assert.LessOrEqual(edge.Destination.Id, expectedNodeCount)
if edge.Destination.Entry.Name == UnresolvedNodeName {
assert.Equal(edge.Destination.Name, edge.Destination.Entry.IP)
} else {
assert.Equal(edge.Destination.Name, edge.Destination.Entry.Name)
}
assert.Equal(destEntryName, edge.Destination.Entry.Name)
// protocol
assert.Equal(protocolName, edge.Protocol.Name)
assert.Equal(protocolCount, edge.Count)
}
for i, v := range edges {
if v.Source.Entry.Name == a && v.Destination.Entry.Name == b && v.Protocol.Name == "http" {
validateEdge(v, a, b, ProtocolHttp.Name, 2)
abEdge = i
continue
}
if v.Source.Entry.Name == a && v.Destination.Entry.Name == b && v.Protocol.Name == "redis" {
validateEdge(v, a, b, ProtocolRedis.Name, 1)
abEdge = i
continue
}
if v.Source.Entry.Name == UnresolvedNodeName && v.Destination.Entry.Name == a {
validateEdge(v, UnresolvedNodeName, a, ProtocolHttp.Name, 1)
uaEdge = i
continue
}
if v.Source.Entry.Name == b && v.Destination.Entry.Name == UnresolvedNodeName {
validateEdge(v, b, UnresolvedNodeName, ProtocolHttp.Name, 1)
buEdge = i
continue
}
if v.Source.Entry.Name == c && v.Destination.Entry.Name == d {
validateEdge(v, c, d, ProtocolHttp.Name, 1)
cdEdge = i
continue
}
if v.Source.Entry.Name == a && v.Destination.Entry.Name == c {
validateEdge(v, a, c, ProtocolHttp.Name, 1)
acEdge = i
continue
}
}
// Make sure we found all the edges
for _, v := range [...]int{abEdge, uaEdge, buEdge, cdEdge, acEdge} {
assert.NotEqual(-1, v)
}
// Reset
s.instance.Reset()
status = s.instance.GetStatus()
nodes = s.instance.GetNodes()
edges = s.instance.GetEdges()
// Counts after reset
assert.Equal(0, s.instance.GetEntriesProcessedCount())
assert.Equal(0, s.instance.GetNodesCount())
assert.Equal(0, s.instance.GetEdgesCount())
// Status after reset
assert.Equal("enabled", status.Status)
assert.Equal(0, status.EntriesProcessedCount)
assert.Equal(0, status.NodeCount)
assert.Equal(0, status.EdgeCount)
// Nodes after reset
assert.Equal([]ServiceMapNode{}, nodes)
// Edges after reset
assert.Equal([]ServiceMapEdge{}, edges)
}
func TestServiceMapSuite(t *testing.T) {
suite.Run(t, new(ServiceMapDisabledSuite))
suite.Run(t, new(ServiceMapEnabledSuite))
}

49
agent/pkg/utils/pathUtils.go
View File

@ -1,49 +0,0 @@
package utils
import (
"os"
"github.com/djherbis/atime"
)
type ByModTime []os.FileInfo
func (fis ByModTime) Len() int {
return len(fis)
}
func (fis ByModTime) Swap(i, j int) {
fis[i], fis[j] = fis[j], fis[i]
}
func (fis ByModTime) Less(i, j int) bool {
return fis[i].ModTime().Before(fis[j].ModTime())
}
type ByName []os.FileInfo
func (fis ByName) Len() int {
return len(fis)
}
func (fis ByName) Swap(i, j int) {
fis[i], fis[j] = fis[j], fis[i]
}
func (fis ByName) Less(i, j int) bool {
return fis[i].Name() < fis[j].Name()
}
type ByCreationTime []os.FileInfo
func (fis ByCreationTime) Len() int {
return len(fis)
}
func (fis ByCreationTime) Swap(i, j int) {
fis[i], fis[j] = fis[j], fis[i]
}
func (fis ByCreationTime) Less(i, j int) bool {
return atime.Get(fis[i]).Unix() < atime.Get(fis[j]).Unix()
}

96
agent/pkg/utils/utils.go
View File

@ -1,96 +0,0 @@
package utils
import (
"context"
"encoding/json"
"fmt"
"net/http"
"os"
"os/signal"
"syscall"
"time"
"github.com/gin-gonic/gin"
"github.com/kubeshark/kubeshark/logger"
)
var (
StartTime int64 // global
)
// StartServer starts the server with a graceful shutdown
func StartServer(app *gin.Engine, port int) {
signals := make(chan os.Signal, 2)
signal.Notify(signals,
os.Interrupt, // this catch ctrl + c
syscall.SIGTSTP, // this catch ctrl + z
)
srv := &http.Server{
Addr: ":8080",
Handler: app,
}
go func() {
<-signals
logger.Log.Infof("Shutting down...")
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
err := srv.Shutdown(ctx)
if err != nil {
logger.Log.Errorf("%v", err)
}
os.Exit(0)
}()
// Run server.
logger.Log.Infof("Starting the server...")
if err := app.Run(fmt.Sprintf(":%d", port)); err != nil {
logger.Log.Errorf("Server is not running! Reason: %v", err)
}
}
func CheckErr(e error) {
if e != nil {
logger.Log.Errorf("%v", e)
}
}
func ReadJsonFile(filePath string, value interface{}) error {
if content, err := os.ReadFile(filePath); err != nil {
return err
} else {
if err = json.Unmarshal(content, value); err != nil {
return err
}
}
return nil
}
func SaveJsonFile(filePath string, value interface{}) error {
if data, err := json.Marshal(value); err != nil {
return err
} else {
if err = os.WriteFile(filePath, data, 0644); err != nil {
return err
}
}
return nil
}
func UniqueStringSlice(s []string) []string {
uniqueSlice := make([]string, 0)
uniqueMap := map[string]bool{}
for _, val := range s {
if uniqueMap[val] {
continue
}
uniqueMap[val] = true
uniqueSlice = append(uniqueSlice, val)
}
return uniqueSlice
}

35
agent/pkg/validation/validation.go
View File

@ -1,35 +0,0 @@
package validation
import (
"fmt"
"github.com/go-playground/locales/en"
ut "github.com/go-playground/universal-translator"
"github.com/go-playground/validator/v10"
et "github.com/go-playground/validator/v10/translations/en"
)
func Validate(object interface{}) (errs []string){
validate, trans := getValidator()
err := validate.Struct(object)
return translateError(err, trans)
}
func translateError(err error, trans *ut.Translator) (errs []string) {
if err == nil {
return nil
}
validatorErrs := err.(validator.ValidationErrors)
for _, e := range validatorErrs {
translatedErr := fmt.Errorf(e.Translate(*trans)).Error()
errs = append(errs, translatedErr)
}
return errs
}
func getValidator() (*validator.Validate, *ut.Translator) {
validate := validator.New()
english := en.New()
trans, _ := ut.New(english, english).GetTranslator("en")
_ = et.RegisterDefaultTranslations(validate, trans)
return validate, &trans
}

8
agent/pkg/version/consts.go
View File

@ -1,8 +0,0 @@
package version
var (
Ver = "0.0.1"
Branch = "develop"
GitCommitHash = "" // this var is overridden using ldflags in makefile when building
BuildTimestamp = "" // this var is overridden using ldflags in makefile when building
)

2
shared/consts.go
View File

@ -8,7 +8,7 @@ const (
DataDirPath = "/app/data/"
ConfigFileName = "kubeshark-config.json"
LogLevelEnvVar = "LOG_LEVEL"
KubesharkAgentImageRepo = "docker.io/kubeshark/kubeshark"
KubesharkAgentImageRepo = "docker.io/kubeshark/hub"
BasenineHost = "127.0.0.1"
BaseninePort = "9099"
BasenineReconnectInterval = 3

2
shared/kubernetes/consts.go
View File

@ -2,7 +2,7 @@ package kubernetes
const (
KubesharkResourcesPrefix = "ks-"
ApiServerPodName = KubesharkResourcesPrefix + "api-server"
ApiServerPodName = KubesharkResourcesPrefix + "hub"
ClusterRoleBindingName = KubesharkResourcesPrefix + "cluster-role-binding"
ClusterRoleName = KubesharkResourcesPrefix + "cluster-role"
K8sAllNamespaces = ""

2
shared/kubernetes/provider.go
View File

@ -206,7 +206,7 @@ func (provider *Provider) BuildApiServerPod(opts *ApiServerOptions, mountVolumeC
}
command := []string{
"./kubesharkagent",
"./hub",
}
if opts.Profiler {