diff --git a/Dockerfile b/Dockerfile index 52d2056b6..4bbdf3984 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,8 +2,10 @@ FROM node:14-slim AS site-build WORKDIR /app/ui-build -COPY ui . +COPY ui/package.json . +COPY ui/package-lock.json . RUN npm i +COPY ui . RUN npm run build diff --git a/README.md b/README.md index 072e91e90..aa9a73e56 100644 --- a/README.md +++ b/README.md @@ -159,6 +159,13 @@ Such validation may test response for specific JSON fields, headers, etc. Please see [TRAFFIC RULES](docs/POLICY_RULES.md) page for more details and syntax. +### OpenAPI Specification (OAS) Contract Monitoring + +An OAS/Swagger file can contain schemas under `parameters` and `responses` fields. With `--contract catalogue.yaml` +CLI option, you can pass your API description to Mizu and the traffic will automatically be validated +against the contracts. + +Please see [CONTRACT MONITORING](docs/CONTRACT_MONITORING.md) page for more details and syntax. ## How to Run local UI diff --git a/agent/go.mod b/agent/go.mod index dc35d88b3..d84808e0e 100644 --- a/agent/go.mod +++ b/agent/go.mod @@ -5,6 +5,7 @@ go 1.16 require ( github.com/djherbis/atime v1.0.0 github.com/fsnotify/fsnotify v1.4.9 + github.com/getkin/kin-openapi v0.76.0 github.com/gin-contrib/static v0.0.1 github.com/gin-gonic/gin v1.7.2 github.com/go-playground/locales v0.13.0 diff --git a/agent/go.sum b/agent/go.sum index 001c34f41..359a91043 100644 --- a/agent/go.sum +++ b/agent/go.sum @@ -68,6 +68,10 @@ github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoD github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/getkin/kin-openapi v0.76.0 h1:j77zg3Ec+k+r+GA3d8hBoXpAc6KX9TbBPrwQGBIy2sY= +github.com/getkin/kin-openapi v0.76.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSyhcnluiMv+Xg= +github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= github.com/gin-contrib/static v0.0.1 h1:JVxuvHPuUfkoul12N7dtQw7KRn/pSMq7Ue1Va9Swm1U= @@ -83,10 +87,13 @@ github.com/go-logr/logr v0.4.0 h1:K7/B1jt6fIBQVd4Owv2MqGQClcgf0R266+7C/QjRcLc= github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= +github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/swag v0.19.5 h1:lTz6Ys4CmqqCQmZPBlbQENR1/GucA2bzYTE12Pw4tFY= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A= github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= @@ -178,6 +185,8 @@ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+ github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gnostic v0.4.1 h1:DLJCy1n/vrD4HPjOvYcT8aYQXpPIzoRZONaYwyycI+I= github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= +github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= +github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= @@ -216,6 +225,7 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e h1:hB2xlXdHp/pmPZq0y3QnmWAArdw9PqbmotexnWx/FU8= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE= github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= @@ -274,6 +284,7 @@ github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoH github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= @@ -532,6 +543,7 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWD gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/agent/pkg/api/contract_validation.go b/agent/pkg/api/contract_validation.go new file mode 100644 index 000000000..e98e165df --- /dev/null +++ b/agent/pkg/api/contract_validation.go @@ -0,0 +1,110 @@ +package api + +import ( + "bytes" + "context" + "fmt" + "io/ioutil" + "net/http" + + "github.com/getkin/kin-openapi/openapi3" + "github.com/getkin/kin-openapi/openapi3filter" + "github.com/getkin/kin-openapi/routers" + legacyrouter "github.com/getkin/kin-openapi/routers/legacy" + "github.com/romana/rlog" + + "github.com/up9inc/mizu/shared" + "github.com/up9inc/mizu/tap/api" +) + +const ( + ContractNotApplicable api.ContractStatus = 0 + ContractPassed api.ContractStatus = 1 + ContractFailed api.ContractStatus = 2 +) + +func loadOAS(ctx context.Context) (doc *openapi3.T, contractContent string, router routers.Router, err error) { + path := fmt.Sprintf("%s/%s", shared.RulePolicyPath, shared.ContractFileName) + bytes, err := ioutil.ReadFile(path) + if err != nil { + rlog.Error(err.Error()) + return + } + contractContent = string(bytes) + loader := &openapi3.Loader{Context: ctx} + doc, _ = loader.LoadFromData(bytes) + err = doc.Validate(ctx) + if err != nil { + rlog.Error(err.Error()) + return + } + router, _ = legacyrouter.NewRouter(doc) + return +} + +func validateOAS(ctx context.Context, doc *openapi3.T, router routers.Router, req *http.Request, res *http.Response) (isValid bool, reqErr error, resErr error) { + isValid = true + reqErr = nil + resErr = nil + + // Find route + route, pathParams, err := router.FindRoute(req) + if err != nil { + return + } + + // Validate request + requestValidationInput := &openapi3filter.RequestValidationInput{ + Request: req, + PathParams: pathParams, + Route: route, + } + if reqErr = openapi3filter.ValidateRequest(ctx, requestValidationInput); reqErr != nil { + isValid = false + } + + responseValidationInput := &openapi3filter.ResponseValidationInput{ + RequestValidationInput: requestValidationInput, + Status: res.StatusCode, + Header: res.Header, + } + + if res.Body != nil { + body, _ := ioutil.ReadAll(res.Body) + res.Body = ioutil.NopCloser(bytes.NewBuffer(body)) + responseValidationInput.SetBodyBytes(body) + } + + // Validate response. + if resErr = openapi3filter.ValidateResponse(ctx, responseValidationInput); resErr != nil { + isValid = false + } + + return +} + +func handleOAS(ctx context.Context, doc *openapi3.T, router routers.Router, req *http.Request, res *http.Response, contractContent string) (contract api.Contract) { + contract = api.Contract{ + Content: contractContent, + Status: ContractNotApplicable, + } + + isValid, reqErr, resErr := validateOAS(ctx, doc, router, req, res) + if isValid { + contract.Status = ContractPassed + } else { + contract.Status = ContractFailed + if reqErr != nil { + contract.RequestReason = reqErr.Error() + } else { + contract.RequestReason = "" + } + if resErr != nil { + contract.ResponseReason = resErr.Error() + } else { + contract.ResponseReason = "" + } + } + + return +} diff --git a/agent/pkg/api/main.go b/agent/pkg/api/main.go index 210d3cefb..56968ff3c 100644 --- a/agent/pkg/api/main.go +++ b/agent/pkg/api/main.go @@ -99,6 +99,14 @@ func startReadingChannel(outputItems <-chan *tapApi.OutputChannelItem, extension panic("Channel of captured messages is nil") } + disableOASValidation := false + ctx := context.Background() + doc, contractContent, router, err := loadOAS(ctx) + if err != nil { + logger.Log.Infof("Disabled OAS validation: %s\n", err.Error()) + disableOASValidation = true + } + for item := range outputItems { providers.EntryAdded() @@ -107,8 +115,19 @@ func startReadingChannel(outputItems <-chan *tapApi.OutputChannelItem, extension mizuEntry := extension.Dissector.Analyze(item, primitive.NewObjectID().Hex(), resolvedSource, resolvedDestionation) baseEntry := extension.Dissector.Summarize(mizuEntry) mizuEntry.EstimatedSizeBytes = getEstimatedEntrySizeBytes(mizuEntry) - database.CreateEntry(mizuEntry) if extension.Protocol.Name == "http" { + if !disableOASValidation { + var httpPair tapApi.HTTPRequestResponsePair + json.Unmarshal([]byte(mizuEntry.Entry), &httpPair) + + contract := handleOAS(ctx, doc, router, httpPair.Request.Payload.RawRequest, httpPair.Response.Payload.RawResponse, contractContent) + baseEntry.ContractStatus = contract.Status + mizuEntry.ContractStatus = contract.Status + mizuEntry.ContractRequestReason = contract.RequestReason + mizuEntry.ContractResponseReason = contract.ResponseReason + mizuEntry.ContractContent = contract.Content + } + var pair tapApi.RequestResponsePair json.Unmarshal([]byte(mizuEntry.Entry), &pair) harEntry, err := utils.NewEntry(&pair) @@ -117,6 +136,7 @@ func startReadingChannel(outputItems <-chan *tapApi.OutputChannelItem, extension baseEntry.Rules = rules } } + database.CreateEntry(mizuEntry) baseEntryBytes, _ := models.CreateBaseEntryWebSocketMessage(baseEntry) BroadcastToBrowserClients(baseEntryBytes) diff --git a/agent/pkg/rules/rulesHTTP.go b/agent/pkg/rules/rulesHTTP.go index 3b4007b87..1313319e6 100644 --- a/agent/pkg/rules/rulesHTTP.go +++ b/agent/pkg/rules/rulesHTTP.go @@ -46,7 +46,7 @@ func ValidateService(serviceFromRule string, service string) bool { func MatchRequestPolicy(harEntry har.Entry, service string) (resultPolicyToSend []RulesMatched, isEnabled bool) { enforcePolicy, err := shared.DecodeEnforcePolicy(fmt.Sprintf("%s/%s", shared.RulePolicyPath, shared.RulePolicyFileName)) - if err == nil { + if err == nil && len(enforcePolicy.Rules) > 0 { isEnabled = true } for _, rule := range enforcePolicy.Rules { diff --git a/cli/cmd/tap.go b/cli/cmd/tap.go index e809b3b6a..82aa8d545 100644 --- a/cli/cmd/tap.go +++ b/cli/cmd/tap.go @@ -102,4 +102,5 @@ func init() { tapCmd.Flags().Bool(configStructs.DryRunTapName, defaultTapConfig.DryRun, "Preview of all pods matching the regex, without tapping them") tapCmd.Flags().StringP(configStructs.WorkspaceTapName, "w", defaultTapConfig.Workspace, "Uploads traffic to your UP9 workspace for further analysis (requires auth)") tapCmd.Flags().String(configStructs.EnforcePolicyFile, defaultTapConfig.EnforcePolicyFile, "Yaml file path with policy rules") + tapCmd.Flags().String(configStructs.ContractFile, defaultTapConfig.ContractFile, "OAS/Swagger file to validate to monitor the contracts") } diff --git a/cli/cmd/tapRunner.go b/cli/cmd/tapRunner.go index 1b445a3cc..2909fb583 100644 --- a/cli/cmd/tapRunner.go +++ b/cli/cmd/tapRunner.go @@ -3,6 +3,7 @@ package cmd import ( "context" "fmt" + "io/ioutil" "path" "regexp" "strings" @@ -12,6 +13,7 @@ import ( core "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/wait" + "github.com/getkin/kin-openapi/openapi3" "github.com/up9inc/mizu/cli/apiserver" "github.com/up9inc/mizu/cli/config" "github.com/up9inc/mizu/cli/config/configStructs" @@ -58,6 +60,30 @@ func RunMizuTap() { } } + // Read and validate the OAS file + var contract string + if config.Config.Tap.ContractFile != "" { + bytes, err := ioutil.ReadFile(config.Config.Tap.ContractFile) + if err != nil { + logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error reading contract file: %v", errormessage.FormatError(err))) + return + } + contract = string(bytes) + + ctx := context.Background() + loader := &openapi3.Loader{Context: ctx} + doc, err := loader.LoadFromData(bytes) + if err != nil { + logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error loading contract file: %v", errormessage.FormatError(err))) + return + } + err = doc.Validate(ctx) + if err != nil { + logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error validating contract file: %v", errormessage.FormatError(err))) + return + } + } + kubernetesProvider, err := kubernetes.NewProvider(config.Config.KubeConfigPath()) if err != nil { logger.Log.Error(err) @@ -104,7 +130,7 @@ func RunMizuTap() { } defer finishMizuExecution(kubernetesProvider) - if err := createMizuResources(ctx, kubernetesProvider, mizuValidationRules); err != nil { + if err := createMizuResources(ctx, kubernetesProvider, mizuValidationRules, contract); err != nil { logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error creating resources: %v", errormessage.FormatError(err))) return } @@ -126,7 +152,7 @@ func readValidationRules(file string) (string, error) { return string(newContent), nil } -func createMizuResources(ctx context.Context, kubernetesProvider *kubernetes.Provider, mizuValidationRules string) error { +func createMizuResources(ctx context.Context, kubernetesProvider *kubernetes.Provider, mizuValidationRules string, contract string) error { if !config.Config.IsNsRestrictedMode() { if err := createMizuNamespace(ctx, kubernetesProvider); err != nil { return err @@ -137,15 +163,15 @@ func createMizuResources(ctx context.Context, kubernetesProvider *kubernetes.Pro return err } - if err := createMizuConfigmap(ctx, kubernetesProvider, mizuValidationRules); err != nil { + if err := createMizuConfigmap(ctx, kubernetesProvider, mizuValidationRules, contract); err != nil { logger.Log.Warningf(uiUtils.Warning, fmt.Sprintf("Failed to create resources required for policy validation. Mizu will not validate policy rules. error: %v\n", errormessage.FormatError(err))) } return nil } -func createMizuConfigmap(ctx context.Context, kubernetesProvider *kubernetes.Provider, data string) error { - err := kubernetesProvider.CreateConfigMap(ctx, config.Config.MizuResourcesNamespace, mizu.ConfigMapName, data) +func createMizuConfigmap(ctx context.Context, kubernetesProvider *kubernetes.Provider, data string, contract string) error { + err := kubernetesProvider.CreateConfigMap(ctx, config.Config.MizuResourcesNamespace, mizu.ConfigMapName, data, contract) return err } diff --git a/cli/config/configStructs/tapConfig.go b/cli/config/configStructs/tapConfig.go index 7e6aca356..35a71c9fd 100644 --- a/cli/config/configStructs/tapConfig.go +++ b/cli/config/configStructs/tapConfig.go @@ -3,8 +3,9 @@ package configStructs import ( "errors" "fmt" - "github.com/up9inc/mizu/shared/units" "regexp" + + "github.com/up9inc/mizu/shared/units" ) const ( @@ -18,6 +19,7 @@ const ( DryRunTapName = "dry-run" WorkspaceTapName = "workspace" EnforcePolicyFile = "traffic-validation-file" + ContractFile = "contract" ) type TapConfig struct { @@ -34,6 +36,7 @@ type TapConfig struct { DryRun bool `yaml:"dry-run" default:"false"` Workspace string `yaml:"workspace"` EnforcePolicyFile string `yaml:"traffic-validation-file"` + ContractFile string `yaml:"contract"` ApiServerResources Resources `yaml:"api-server-resources"` TapperResources Resources `yaml:"tapper-resources"` } diff --git a/cli/go.mod b/cli/go.mod index 11c0f116a..c2ded4547 100644 --- a/cli/go.mod +++ b/cli/go.mod @@ -5,6 +5,7 @@ go 1.16 require ( github.com/creasty/defaults v1.5.1 github.com/denisbrodbeck/machineid v1.0.1 + github.com/getkin/kin-openapi v0.79.0 github.com/google/go-github/v37 v37.0.0 github.com/google/uuid v1.1.2 github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 diff --git a/cli/go.sum b/cli/go.sum index 843dde79e..7d815da49 100644 --- a/cli/go.sum +++ b/cli/go.sum @@ -113,6 +113,9 @@ github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoD github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fvbommel/sortorder v1.0.1/go.mod h1:uk88iVf1ovNn1iLfgUVU2F9o5eO30ui720w+kxuqRs0= +github.com/getkin/kin-openapi v0.79.0 h1:YLZIgIhZLq9z5WFHHIK+oWORRfn6jjwr7qN0xak0xbE= +github.com/getkin/kin-openapi v0.79.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSyhcnluiMv+Xg= +github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= @@ -140,6 +143,8 @@ github.com/go-openapi/jsonpointer v0.17.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwds github.com/go-openapi/jsonpointer v0.18.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= +github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonreference v0.17.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= github.com/go-openapi/jsonreference v0.18.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= @@ -165,6 +170,7 @@ github.com/go-openapi/strfmt v0.19.5/go.mod h1:eftuHTlB/dI8Uq8JJOyRlieZf+WkkxUuk github.com/go-openapi/swag v0.17.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= github.com/go-openapi/swag v0.18.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/swag v0.19.5 h1:lTz6Ys4CmqqCQmZPBlbQENR1/GucA2bzYTE12Pw4tFY= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/validate v0.18.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4= github.com/go-openapi/validate v0.19.2/go.mod h1:1tRCw7m3jtI8eNWEEliiAqUIcBztB2KDnRCRMUi7GTA= @@ -221,6 +227,7 @@ github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= @@ -238,6 +245,7 @@ github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5m github.com/googleapis/gnostic v0.4.1 h1:DLJCy1n/vrD4HPjOvYcT8aYQXpPIzoRZONaYwyycI+I= github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= @@ -299,6 +307,7 @@ github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.7.0 h1:aizVhC/NAAcKWb+5QsU1iNOZb4Yws5UO2I+aIprQITM= github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= github.com/markbates/pkger v0.17.1/go.mod h1:0JoVlrol20BSywW79rN3kdFFsE5xYM+rSCQDXbLhiuI= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= @@ -372,6 +381,8 @@ github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4O github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/romana/rlog v0.0.0-20171115192701-f018bc92e7d7 h1:jkvpcEatpwuMF5O5LVxTnehj6YZ/aEZN4NWD/Xml4pI= +github.com/romana/rlog v0.0.0-20171115192701-f018bc92e7d7/go.mod h1:KTrHyWpO1sevuXPZwyeZc72ddWRFqNSKDFl7uVWKpg0= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= @@ -405,6 +416,7 @@ github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoH github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= @@ -692,6 +704,7 @@ gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/cli/kubernetes/provider.go b/cli/kubernetes/provider.go index 3cae70685..4cf36c3ca 100644 --- a/cli/kubernetes/provider.go +++ b/cli/kubernetes/provider.go @@ -485,13 +485,14 @@ func (provider *Provider) handleRemovalError(err error) error { return err } -func (provider *Provider) CreateConfigMap(ctx context.Context, namespace string, configMapName string, data string) error { - if data == "" { +func (provider *Provider) CreateConfigMap(ctx context.Context, namespace string, configMapName string, data string, contract string) error { + if data == "" && contract == "" { return nil } configMapData := make(map[string]string, 0) configMapData[shared.RulePolicyFileName] = data + configMapData[shared.ContractFileName] = contract configMap := &core.ConfigMap{ TypeMeta: metav1.TypeMeta{ Kind: "ConfigMap", diff --git a/docs/CONTRACT_MONITORING.md b/docs/CONTRACT_MONITORING.md new file mode 100644 index 000000000..41b31ccf2 --- /dev/null +++ b/docs/CONTRACT_MONITORING.md @@ -0,0 +1,172 @@ +# OpenAPI Specification (OAS) Contract Monitoring + +An OAS/Swagger file can contain schemas under `parameters` and `responses` fields. With `--contract catalogue.yaml` +CLI option, you can pass your API description to Mizu and the traffic will automatically be validated +against the contracts. + +Below is an example of an OAS/Swagger file from [Sock Shop](https://microservices-demo.github.io/) microservice demo +that contains a bunch contracts: + +```yaml +openapi: 3.0.1 +info: + title: Catalogue resources + version: 1.0.0 + description: "" + license: + name: MIT + url: http://github.com/gruntjs/grunt/blob/master/LICENSE-MIT +paths: + /catalogue: + get: + description: Catalogue API + operationId: List catalogue + responses: + 200: + description: "" + content: + application/json;charset=UTF-8: + schema: + type: array + items: + $ref: '#/components/schemas/Listresponse' + /catalogue/{id}: + get: + operationId: Get an item + parameters: + - name: id + in: path + required: true + schema: + type: string + example: a0a4f044-b040-410d-8ead-4de0446aec7e + responses: + 200: + description: "" + content: + application/json; charset=UTF-8: + schema: + $ref: '#/components/schemas/Getanitemresponse' + /catalogue/size: + get: + operationId: Get size + responses: + 200: + description: "" + content: + application/json;charset=UTF-8: + schema: + $ref: '#/components/schemas/Getsizeresponse' + /tags: + get: + operationId: List_ + responses: + 200: + description: "" + content: + application/json;charset=UTF-8: + schema: + $ref: '#/components/schemas/Listresponse3' +components: + schemas: + Listresponse: + title: List response + required: + - count + - description + - id + - imageUrl + - name + - price + - tag + type: object + properties: + id: + type: string + name: + type: string + description: + type: string + imageUrl: + type: array + items: + type: string + price: + type: number + format: double + count: + type: integer + format: int32 + tag: + type: array + items: + type: string + Getanitemresponse: + title: Get an item response + required: + - count + - description + - id + - imageUrl + - name + - price + - tag + type: object + properties: + id: + type: string + name: + type: string + description: + type: string + imageUrl: + type: array + items: + type: string + price: + type: number + format: double + count: + type: integer + format: int32 + tag: + type: array + items: + type: string + Getsizeresponse: + title: Get size response + required: + - size + type: object + properties: + size: + type: integer + format: int32 + Listresponse3: + title: List response3 + required: + - tags + type: object + properties: + tags: + type: array + items: + type: string +``` + +Pass it to Mizu through the CLI option: `mizu tap -n sock-shop --contract catalogue.yaml` + +Now Mizu will monitor the traffic against these contracts. + +If an entry fails to comply with the contract, it's marked with `Breach` notice in the UI. +The reason of the failure can be seen under the `CONTRACT` tab in the details layout. + +### Notes + +Make sure that you; + +- specified the `openapi` version +- specified the `info.version` version in the YAML +- and removed `servers` field from the YAML + +Otherwise the OAS file cannot be recognized. (see [this issue](https://github.com/getkin/kin-openapi/issues/356)) diff --git a/shared/consts.go b/shared/consts.go index cf4d6f208..5f4467c1d 100644 --- a/shared/consts.go +++ b/shared/consts.go @@ -9,6 +9,7 @@ const ( MaxEntriesDBSizeBytesEnvVar = "MAX_ENTRIES_DB_BYTES" RulePolicyPath = "/app/enforce-policy/" RulePolicyFileName = "enforce-policy.yaml" + ContractFileName = "contract-oas.yaml" GoGCEnvVar = "GOGC" DefaultApiServerPort = 8899 DebugModeEnvVar = "MIZU_DEBUG" diff --git a/tap/api/api.go b/tap/api/api.go index 94247624f..66de128ec 100644 --- a/tap/api/api.go +++ b/tap/api/api.go @@ -2,9 +2,18 @@ package api import ( "bufio" + "bytes" + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "net/http" "plugin" "sync" "time" + + "github.com/google/martian/har" + "github.com/romana/rlog" ) type Protocol struct { @@ -104,32 +113,36 @@ type MizuEntry struct { ID uint `gorm:"primarykey"` CreatedAt time.Time UpdatedAt time.Time - ProtocolName string `json:"protocolName" gorm:"column:protocolName"` - ProtocolLongName string `json:"protocolLongName" gorm:"column:protocolLongName"` - ProtocolAbbreviation string `json:"protocolAbbreviation" gorm:"column:protocolAbbreviation"` - ProtocolVersion string `json:"protocolVersion" gorm:"column:protocolVersion"` - ProtocolBackgroundColor string `json:"protocolBackgroundColor" gorm:"column:protocolBackgroundColor"` - ProtocolForegroundColor string `json:"protocolForegroundColor" gorm:"column:protocolForegroundColor"` - ProtocolFontSize int8 `json:"protocolFontSize" gorm:"column:protocolFontSize"` - ProtocolReferenceLink string `json:"protocolReferenceLink" gorm:"column:protocolReferenceLink"` - Entry string `json:"entry,omitempty" gorm:"column:entry"` - EntryId string `json:"entryId" gorm:"column:entryId"` - Url string `json:"url" gorm:"column:url"` - Method string `json:"method" gorm:"column:method"` - Status int `json:"status" gorm:"column:status"` - RequestSenderIp string `json:"requestSenderIp" gorm:"column:requestSenderIp"` - Service string `json:"service" gorm:"column:service"` - Timestamp int64 `json:"timestamp" gorm:"column:timestamp"` - ElapsedTime int64 `json:"elapsedTime" gorm:"column:elapsedTime"` - Path string `json:"path" gorm:"column:path"` - ResolvedSource string `json:"resolvedSource,omitempty" gorm:"column:resolvedSource"` - ResolvedDestination string `json:"resolvedDestination,omitempty" gorm:"column:resolvedDestination"` - SourceIp string `json:"sourceIp,omitempty" gorm:"column:sourceIp"` - DestinationIp string `json:"destinationIp,omitempty" gorm:"column:destinationIp"` - SourcePort string `json:"sourcePort,omitempty" gorm:"column:sourcePort"` - DestinationPort string `json:"destinationPort,omitempty" gorm:"column:destinationPort"` - IsOutgoing bool `json:"isOutgoing,omitempty" gorm:"column:isOutgoing"` - EstimatedSizeBytes int `json:"-" gorm:"column:estimatedSizeBytes"` + ProtocolName string `json:"protocolName" gorm:"column:protocolName"` + ProtocolLongName string `json:"protocolLongName" gorm:"column:protocolLongName"` + ProtocolAbbreviation string `json:"protocolAbbreviation" gorm:"column:protocolAbbreviation"` + ProtocolVersion string `json:"protocolVersion" gorm:"column:protocolVersion"` + ProtocolBackgroundColor string `json:"protocolBackgroundColor" gorm:"column:protocolBackgroundColor"` + ProtocolForegroundColor string `json:"protocolForegroundColor" gorm:"column:protocolForegroundColor"` + ProtocolFontSize int8 `json:"protocolFontSize" gorm:"column:protocolFontSize"` + ProtocolReferenceLink string `json:"protocolReferenceLink" gorm:"column:protocolReferenceLink"` + Entry string `json:"entry,omitempty" gorm:"column:entry"` + EntryId string `json:"entryId" gorm:"column:entryId"` + Url string `json:"url" gorm:"column:url"` + Method string `json:"method" gorm:"column:method"` + Status int `json:"status" gorm:"column:status"` + RequestSenderIp string `json:"requestSenderIp" gorm:"column:requestSenderIp"` + Service string `json:"service" gorm:"column:service"` + Timestamp int64 `json:"timestamp" gorm:"column:timestamp"` + ElapsedTime int64 `json:"elapsedTime" gorm:"column:elapsedTime"` + Path string `json:"path" gorm:"column:path"` + ResolvedSource string `json:"resolvedSource,omitempty" gorm:"column:resolvedSource"` + ResolvedDestination string `json:"resolvedDestination,omitempty" gorm:"column:resolvedDestination"` + SourceIp string `json:"sourceIp,omitempty" gorm:"column:sourceIp"` + DestinationIp string `json:"destinationIp,omitempty" gorm:"column:destinationIp"` + SourcePort string `json:"sourcePort,omitempty" gorm:"column:sourcePort"` + DestinationPort string `json:"destinationPort,omitempty" gorm:"column:destinationPort"` + IsOutgoing bool `json:"isOutgoing,omitempty" gorm:"column:isOutgoing"` + ContractStatus ContractStatus `json:"contractStatus,omitempty" gorm:"column:contractStatus"` + ContractRequestReason string `json:"contractRequestReason,omitempty" gorm:"column:contractRequestReason"` + ContractResponseReason string `json:"contractResponseReason,omitempty" gorm:"column:contractResponseReason"` + ContractContent string `json:"contractContent,omitempty" gorm:"column:contractContent"` + EstimatedSizeBytes int `json:"-" gorm:"column:estimatedSizeBytes"` } type MizuEntryWrapper struct { @@ -159,6 +172,7 @@ type BaseEntryDetails struct { IsOutgoing bool `json:"isOutgoing,omitempty"` Latency int64 `json:"latency"` Rules ApplicableRules `json:"rules,omitempty"` + ContractStatus ContractStatus `json:"contractStatus"` } type ApplicableRules struct { @@ -167,6 +181,15 @@ type ApplicableRules struct { NumberOfRules int `json:"numberOfRules,omitempty"` } +type ContractStatus int + +type Contract struct { + Status ContractStatus `json:"status"` + RequestReason string `json:"requestReason"` + ResponseReason string `json:"responseReason"` + Content string `json:"content"` +} + type DataUnmarshaler interface { UnmarshalData(*MizuEntry) error } @@ -192,6 +215,7 @@ func (bed *BaseEntryDetails) UnmarshalData(entry *MizuEntry) error { bed.RequestSenderIp = entry.RequestSenderIp bed.IsOutgoing = entry.IsOutgoing bed.Latency = entry.ElapsedTime + bed.ContractStatus = entry.ContractStatus return nil } @@ -199,3 +223,111 @@ const ( TABLE string = "table" BODY string = "body" ) + +const ( + TypeHttpRequest = iota + TypeHttpResponse +) + +type HTTPPayload struct { + Type uint8 + Data interface{} +} + +type HTTPPayloader interface { + MarshalJSON() ([]byte, error) +} + +type HTTPWrapper struct { + Method string `json:"method"` + Url string `json:"url"` + Details interface{} `json:"details"` + RawRequest *HTTPRequestWrapper `json:"rawRequest"` + RawResponse *HTTPResponseWrapper `json:"rawResponse"` +} + +func (h HTTPPayload) MarshalJSON() ([]byte, error) { + switch h.Type { + case TypeHttpRequest: + harRequest, err := har.NewRequest(h.Data.(*http.Request), true) + if err != nil { + rlog.Debugf("convert-request-to-har", "Failed converting request to HAR %s (%v,%+v)", err, err, err) + return nil, errors.New("Failed converting request to HAR") + } + return json.Marshal(&HTTPWrapper{ + Method: harRequest.Method, + Url: "", + Details: harRequest, + RawRequest: &HTTPRequestWrapper{Request: h.Data.(*http.Request)}, + }) + case TypeHttpResponse: + harResponse, err := har.NewResponse(h.Data.(*http.Response), true) + if err != nil { + rlog.Debugf("convert-response-to-har", "Failed converting response to HAR %s (%v,%+v)", err, err, err) + return nil, errors.New("Failed converting response to HAR") + } + return json.Marshal(&HTTPWrapper{ + Method: "", + Url: "", + Details: harResponse, + RawResponse: &HTTPResponseWrapper{Response: h.Data.(*http.Response)}, + }) + default: + panic(fmt.Sprintf("HTTP payload cannot be marshaled: %s\n", h.Type)) + } +} + +type HTTPWrapperTricky struct { + Method string `json:"method"` + Url string `json:"url"` + Details interface{} `json:"details"` + RawRequest *http.Request `json:"rawRequest"` + RawResponse *http.Response `json:"rawResponse"` +} + +type HTTPMessage struct { + IsRequest bool `json:"isRequest"` + CaptureTime time.Time `json:"captureTime"` + Payload HTTPWrapperTricky `json:"payload"` +} + +type HTTPRequestResponsePair struct { + Request HTTPMessage `json:"request"` + Response HTTPMessage `json:"response"` +} + +type HTTPRequestWrapper struct { + *http.Request +} + +func (r *HTTPRequestWrapper) MarshalJSON() ([]byte, error) { + body, _ := ioutil.ReadAll(r.Request.Body) + r.Request.Body = ioutil.NopCloser(bytes.NewBuffer(body)) + return json.Marshal(&struct { + Body string `json:"Body,omitempty"` + GetBody string `json:"GetBody,omitempty"` + Cancel string `json:"Cancel,omitempty"` + *http.Request + }{ + Body: string(body), + Request: r.Request, + }) +} + +type HTTPResponseWrapper struct { + *http.Response +} + +func (r *HTTPResponseWrapper) MarshalJSON() ([]byte, error) { + body, _ := ioutil.ReadAll(r.Response.Body) + r.Response.Body = ioutil.NopCloser(bytes.NewBuffer(body)) + return json.Marshal(&struct { + Body string `json:"Body,omitempty"` + GetBody string `json:"GetBody,omitempty"` + Cancel string `json:"Cancel,omitempty"` + *http.Response + }{ + Body: string(body), + Response: r.Response, + }) +} diff --git a/tap/api/go.mod b/tap/api/go.mod index d5379a1fd..bd49ce403 100644 --- a/tap/api/go.mod +++ b/tap/api/go.mod @@ -1,3 +1,8 @@ module github.com/up9inc/mizu/tap/api go 1.16 + +require ( + github.com/google/martian v2.1.0+incompatible // indirect + github.com/romana/rlog v0.0.0-20171115192701-f018bc92e7d7 // indirect +) diff --git a/tap/api/go.sum b/tap/api/go.sum new file mode 100644 index 000000000..f25f09c37 --- /dev/null +++ b/tap/api/go.sum @@ -0,0 +1,4 @@ +github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/romana/rlog v0.0.0-20171115192701-f018bc92e7d7 h1:jkvpcEatpwuMF5O5LVxTnehj6YZ/aEZN4NWD/Xml4pI= +github.com/romana/rlog v0.0.0-20171115192701-f018bc92e7d7/go.mod h1:KTrHyWpO1sevuXPZwyeZc72ddWRFqNSKDFl7uVWKpg0= diff --git a/tap/extensions/amqp/go.sum b/tap/extensions/amqp/go.sum new file mode 100644 index 000000000..f25f09c37 --- /dev/null +++ b/tap/extensions/amqp/go.sum @@ -0,0 +1,4 @@ +github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/romana/rlog v0.0.0-20171115192701-f018bc92e7d7 h1:jkvpcEatpwuMF5O5LVxTnehj6YZ/aEZN4NWD/Xml4pI= +github.com/romana/rlog v0.0.0-20171115192701-f018bc92e7d7/go.mod h1:KTrHyWpO1sevuXPZwyeZc72ddWRFqNSKDFl7uVWKpg0= diff --git a/tap/extensions/http/matcher.go b/tap/extensions/http/matcher.go index b53793717..588391c40 100644 --- a/tap/extensions/http/matcher.go +++ b/tap/extensions/http/matcher.go @@ -31,7 +31,7 @@ func (matcher *requestResponseMatcher) registerRequest(ident string, request *ht requestHTTPMessage := api.GenericMessage{ IsRequest: true, CaptureTime: captureTime, - Payload: HTTPPayload{ + Payload: api.HTTPPayload{ Type: TypeHttpRequest, Data: request, }, @@ -60,7 +60,7 @@ func (matcher *requestResponseMatcher) registerResponse(ident string, response * responseHTTPMessage := api.GenericMessage{ IsRequest: false, CaptureTime: captureTime, - Payload: HTTPPayload{ + Payload: api.HTTPPayload{ Type: TypeHttpResponse, Data: response, }, diff --git a/tap/extensions/http/sensitive_data_cleaner.go b/tap/extensions/http/sensitive_data_cleaner.go index 9ea0914bf..9c45fb0e4 100644 --- a/tap/extensions/http/sensitive_data_cleaner.go +++ b/tap/extensions/http/sensitive_data_cleaner.go @@ -30,7 +30,7 @@ func IsIgnoredUserAgent(item *api.OutputChannelItem, options *api.TrafficFilteri return false } - request := item.Pair.Request.Payload.(HTTPPayload).Data.(*http.Request) + request := item.Pair.Request.Payload.(api.HTTPPayload).Data.(*http.Request) for headerKey, headerValues := range request.Header { if strings.ToLower(headerKey) == "user-agent" { @@ -50,8 +50,8 @@ func IsIgnoredUserAgent(item *api.OutputChannelItem, options *api.TrafficFilteri } func FilterSensitiveData(item *api.OutputChannelItem, options *api.TrafficFilteringOptions) { - request := item.Pair.Request.Payload.(HTTPPayload).Data.(*http.Request) - response := item.Pair.Response.Payload.(HTTPPayload).Data.(*http.Response) + request := item.Pair.Request.Payload.(api.HTTPPayload).Data.(*http.Request) + response := item.Pair.Response.Payload.(api.HTTPPayload).Data.(*http.Response) filterHeaders(&request.Header) filterHeaders(&response.Header) diff --git a/tap/extensions/http/structs.go b/tap/extensions/http/structs.go deleted file mode 100644 index 6ec868832..000000000 --- a/tap/extensions/http/structs.go +++ /dev/null @@ -1,55 +0,0 @@ -package main - -import ( - "encoding/json" - "errors" - "fmt" - "net/http" - - "github.com/google/martian/har" - "github.com/romana/rlog" -) - -type HTTPPayload struct { - Type uint8 - Data interface{} -} - -type HTTPPayloader interface { - MarshalJSON() ([]byte, error) -} - -type HTTPWrapper struct { - Method string `json:"method"` - Url string `json:"url"` - Details interface{} `json:"details"` -} - -func (h HTTPPayload) MarshalJSON() ([]byte, error) { - switch h.Type { - case TypeHttpRequest: - harRequest, err := har.NewRequest(h.Data.(*http.Request), true) - if err != nil { - rlog.Debugf("convert-request-to-har", "Failed converting request to HAR %s (%v,%+v)", err, err, err) - return nil, errors.New("Failed converting request to HAR") - } - return json.Marshal(&HTTPWrapper{ - Method: harRequest.Method, - Url: "", - Details: harRequest, - }) - case TypeHttpResponse: - harResponse, err := har.NewResponse(h.Data.(*http.Response), true) - if err != nil { - rlog.Debugf("convert-response-to-har", "Failed converting response to HAR %s (%v,%+v)", err, err, err) - return nil, errors.New("Failed converting response to HAR") - } - return json.Marshal(&HTTPWrapper{ - Method: "", - Url: "", - Details: harResponse, - }) - default: - panic(fmt.Sprintf("HTTP payload cannot be marshaled: %s\n", h.Type)) - } -} diff --git a/tap/extensions/kafka/go.sum b/tap/extensions/kafka/go.sum index 32124810d..e0cefe951 100644 --- a/tap/extensions/kafka/go.sum +++ b/tap/extensions/kafka/go.sum @@ -7,6 +7,8 @@ github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/klauspost/compress v1.9.8 h1:VMAMUUOh+gaxKTMk+zqbjsSjsIcUcL/LF4o63i82QyA= github.com/klauspost/compress v1.9.8/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= @@ -17,6 +19,8 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/pierrec/lz4 v2.6.0+incompatible h1:Ix9yFKn1nSPBLFl/yZknTp8TU5G4Ps0JDmguYK6iH1A= github.com/pierrec/lz4 v2.6.0+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/romana/rlog v0.0.0-20171115192701-f018bc92e7d7 h1:jkvpcEatpwuMF5O5LVxTnehj6YZ/aEZN4NWD/Xml4pI= +github.com/romana/rlog v0.0.0-20171115192701-f018bc92e7d7/go.mod h1:KTrHyWpO1sevuXPZwyeZc72ddWRFqNSKDFl7uVWKpg0= github.com/segmentio/kafka-go v0.4.17 h1:IyqRstL9KUTDb3kyGPOOa5VffokKWSEzN6geJ92dSDY= github.com/segmentio/kafka-go v0.4.17/go.mod h1:19+Eg7KwrNKy/PFhiIthEPkO8k+ac7/ZYXwYM9Df10w= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= diff --git a/tap/extensions/redis/go.sum b/tap/extensions/redis/go.sum new file mode 100644 index 000000000..f25f09c37 --- /dev/null +++ b/tap/extensions/redis/go.sum @@ -0,0 +1,4 @@ +github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/romana/rlog v0.0.0-20171115192701-f018bc92e7d7 h1:jkvpcEatpwuMF5O5LVxTnehj6YZ/aEZN4NWD/Xml4pI= +github.com/romana/rlog v0.0.0-20171115192701-f018bc92e7d7/go.mod h1:KTrHyWpO1sevuXPZwyeZc72ddWRFqNSKDFl7uVWKpg0= diff --git a/ui/src/components/EntryDetailed.tsx b/ui/src/components/EntryDetailed.tsx index 65e6ce8b3..ffc5151a9 100644 --- a/ui/src/components/EntryDetailed.tsx +++ b/ui/src/components/EntryDetailed.tsx @@ -71,7 +71,7 @@ export const EntryDetailed: React.FC = ({entryData}) => { /> {entryData.data && } <> - {entryData.data && } + {entryData.data && } }; diff --git a/ui/src/components/EntryDetailed/EntrySections.tsx b/ui/src/components/EntryDetailed/EntrySections.tsx index 586d40d79..88816c596 100644 --- a/ui/src/components/EntryDetailed/EntrySections.tsx +++ b/ui/src/components/EntryDetailed/EntrySections.tsx @@ -150,8 +150,6 @@ export const EntryTableSection: React.FC = ({title, color, ar } - - interface EntryPolicySectionProps { title: string, color: string, @@ -159,7 +157,6 @@ interface EntryPolicySectionProps { arrayToIterate: any[], } - interface EntryPolicySectionCollapsibleTitleProps { label: string; matched: string; @@ -253,3 +250,28 @@ export const EntryTablePolicySection: React.FC = ({titl } } + +interface EntryContractSectionProps { + color: string, + requestReason: string, + responseReason: string, + contractContent: string, +} + +export const EntryContractSection: React.FC = ({color, requestReason, responseReason, contractContent}) => { + return + {requestReason && + {requestReason} + } + {responseReason && + {responseReason} + } + {contractContent && + + } + +} diff --git a/ui/src/components/EntryDetailed/EntryViewer.tsx b/ui/src/components/EntryDetailed/EntryViewer.tsx index 381a554f3..dc8b8f4c7 100644 --- a/ui/src/components/EntryDetailed/EntryViewer.tsx +++ b/ui/src/components/EntryDetailed/EntryViewer.tsx @@ -1,7 +1,7 @@ import React, {useState} from 'react'; import styles from './EntryViewer.module.sass'; import Tabs from "../UI/Tabs"; -import {EntryTableSection, EntryBodySection, EntryTablePolicySection} from "./EntrySections"; +import {EntryTableSection, EntryBodySection, EntryTablePolicySection, EntryContractSection} from "./EntrySections"; enum SectionTypes { SectionTable = "table", @@ -33,7 +33,7 @@ const SectionsRepresentation: React.FC = ({data, color}) => { return <>{sections}; } -const AutoRepresentation: React.FC = ({representation, isRulesEnabled, rulesMatched, elapsedTime, color}) => { +const AutoRepresentation: React.FC = ({representation, isRulesEnabled, rulesMatched, contractStatus, requestReason, responseReason, contractContent, elapsedTime, color}) => { var TABS = [ { tab: 'Request' @@ -50,6 +50,7 @@ const AutoRepresentation: React.FC = ({representation, isRulesEnabled, rule var responseTabIndex = 0; var rulesTabIndex = 0; + var contractTabIndex = 0; if (response) { TABS.push( @@ -69,11 +70,19 @@ const AutoRepresentation: React.FC = ({representation, isRulesEnabled, rule rulesTabIndex = TABS.length - 1; } + if (contractStatus !== 0 && contractContent) { + TABS.push( + { + tab: 'Contract', + } + ); + contractTabIndex = TABS.length - 1; + } + return
{
- {request?.url && {request.payload.url}}
{currentTab === TABS[0].tab && @@ -84,6 +93,9 @@ const AutoRepresentation: React.FC = ({representation, isRulesEnabled, rule {isRulesEnabled && currentTab === TABS[rulesTabIndex].tab && } + {contractStatus !== 0 && contractContent && currentTab === TABS[contractTabIndex].tab && + + }
}
; } @@ -92,12 +104,26 @@ interface Props { representation: any; isRulesEnabled: boolean; rulesMatched: any; + contractStatus: number; + requestReason: string; + responseReason: string; + contractContent: string; color: string; elapsedTime: number; } -const EntryViewer: React.FC = ({representation, isRulesEnabled, rulesMatched, elapsedTime, color}) => { - return +const EntryViewer: React.FC = ({representation, isRulesEnabled, rulesMatched, contractStatus, requestReason, responseReason, contractContent, elapsedTime, color}) => { + return }; export default EntryViewer; diff --git a/ui/src/components/EntryListItem/EntryListItem.module.sass b/ui/src/components/EntryListItem/EntryListItem.module.sass index 2c6ade2c8..d8e5295d5 100644 --- a/ui/src/components/EntryListItem/EntryListItem.module.sass +++ b/ui/src/components/EntryListItem/EntryListItem.module.sass @@ -38,6 +38,7 @@ .ruleNumberText font-size: 12px font-weight: 600 + white-space: nowrap .ruleNumberTextFailure color: #DB2156 @@ -72,12 +73,17 @@ padding-left: 10px flex-grow: 1 -.directionContainer +.separatorRight display: flex border-right: 1px solid $data-background-color padding: 4px padding-right: 12px +.separatorLeft + display: flex + padding: 4px + padding-left: 12px + .port font-size: 12px color: $secondary-font-color diff --git a/ui/src/components/EntryListItem/EntryListItem.tsx b/ui/src/components/EntryListItem/EntryListItem.tsx index 0e8c04546..fc418631f 100644 --- a/ui/src/components/EntryListItem/EntryListItem.tsx +++ b/ui/src/components/EntryListItem/EntryListItem.tsx @@ -26,6 +26,7 @@ interface Entry { isOutgoing?: boolean; latency: number; rules: Rules; + contractStatus: number, } interface Rules { @@ -64,7 +65,7 @@ export const EntryItem: React.FC = ({entry, setFocusedEntryId, isSel } } let additionalRulesProperties = ""; - let ruleSuccess: boolean; + let ruleSuccess = true; let rule = 'latency' in entry.rules if (rule) { if (entry.rules.latency !== -1) { @@ -91,11 +92,32 @@ export const EntryItem: React.FC = ({entry, setFocusedEntryId, isSel } } } + + var contractEnabled = true; + var contractText = ""; + switch (entry.contractStatus) { + case 0: + contractEnabled = false; + break; + case 1: + additionalRulesProperties = styles.ruleSuccessRow + ruleSuccess = true + contractText = "No Breaches" + break; + case 2: + additionalRulesProperties = styles.ruleFailureRow + ruleSuccess = false + contractText = "Breach" + break; + default: + break; + } + return <>
setFocusedEntryId(entry.id)} style={{ border: isSelected ? `1px ${entry.protocol.backgroundColor} solid` : "1px transparent solid", @@ -117,12 +139,19 @@ export const EntryItem: React.FC = ({entry, setFocusedEntryId, isSel
{ rule ? -
+
{`Rules (${numberOfRules})`}
: "" } -
+ { + contractEnabled ? +
+ {contractText} +
+ : "" + } +
{entry.sourcePort} {entry.isOutgoing ? Ingoing traffic