Don't omit the Latency field in BaseEntryDetails (#302)

* .

README.md

@@ -152,12 +152,12 @@ Web interface is now available at http://localhost:8899
 ```
 Any request that contains `User-Agent` header with one of the specified values (`kube-probe` or `prometheus`) will not be captured
 
-### API Rules validation
+### Traffic validation rules
 
-This feature allows you to define set of simple rules, and test the API against them.
+This feature allows you to define set of simple rules, and test the traffic against them.
 Such validation may test response for specific JSON fields, headers, etc.
 
-Please see [API RULES](docs/POLICY_RULES.md) page for more details and syntax.
+Please see [TRAFFIC RULES](docs/POLICY_RULES.md) page for more details and syntax.
 
 
 ## How to Run local UI

agent/main.go

@@ -61,10 +61,10 @@ func main() {
 
 		go filterItems(outputItemsChannel, filteredOutputItemsChannel, filteringOptions)
 		go api.StartReadingEntries(filteredOutputItemsChannel, nil, extensionsMap)
-		// go api.StartReadingOutbound(outboundLinkOutputChannel)
 
 		hostApi(nil)
 	} else if *tapperMode {
+		rlog.Infof("Starting tapper, websocket address: %s", *apiServerAddress)
 		if *apiServerAddress == "" {
 			panic("API server address must be provided with --api-server-address when using --tap")
 		}
@@ -77,13 +77,13 @@ func main() {
 
 		filteredOutputItemsChannel := make(chan *tapApi.OutputChannelItem)
 		tap.StartPassiveTapper(tapOpts, filteredOutputItemsChannel, extensions, filteringOptions)
-		socketConnection, err := shared.ConnectToSocketServer(*apiServerAddress, shared.DEFAULT_SOCKET_RETRIES, shared.DEFAULT_SOCKET_RETRY_SLEEP_TIME, false)
+		socketConnection, err := utils.ConnectToSocketServer(*apiServerAddress)
 		if err != nil {
 			panic(fmt.Sprintf("Error connecting to socket server at %s %v", *apiServerAddress, err))
 		}
+		rlog.Infof("Connected successfully to websocket %s", *apiServerAddress)
 
 		go pipeTapChannelToSocket(socketConnection, filteredOutputItemsChannel)
-		// go pipeOutboundLinksChannelToSocket(socketConnection, outboundLinkOutputChannel)
 	} else if *apiServerMode {
 		api.StartResolving(*namespace)
 
@@ -122,7 +122,7 @@ func loadExtensions() {
 	extensionsMap = make(map[string]*tapApi.Extension)
 	for i, file := range files {
 		filename := file.Name()
-		log.Printf("Loading extension: %s\n", filename)
+		rlog.Infof("Loading extension: %s\n", filename)
 		extension := &tapApi.Extension{
 			Path: path.Join(extensionsDir, filename),
 		}
@@ -290,7 +290,7 @@ func pipeTapChannelToSocket(connection *websocket.Conn, messageDataChannel <-cha
 	for messageData := range messageDataChannel {
 		marshaledData, err := models.CreateWebsocketTappedEntryMessage(messageData)
 		if err != nil {
-			rlog.Infof("error converting message to json %s, (%v,%+v)\n", err, err, err)
+			rlog.Errorf("error converting message to json %v, err: %s, (%v,%+v)", messageData, err, err, err)
 			continue
 		}
 
@@ -298,26 +298,8 @@ func pipeTapChannelToSocket(connection *websocket.Conn, messageDataChannel <-cha
 		// and goes into the intermediate WebSocket.
 		err = connection.WriteMessage(websocket.TextMessage, marshaledData)
 		if err != nil {
-			rlog.Infof("error sending message through socket server %s, (%v,%+v)\n", err, err, err)
+			rlog.Errorf("error sending message through socket server %v, err: %s, (%v,%+v)", messageData, err, err, err)
 			continue
 		}
 	}
 }
-
-func pipeOutboundLinksChannelToSocket(connection *websocket.Conn, outboundLinkChannel <-chan *tap.OutboundLink) {
-	for outboundLink := range outboundLinkChannel {
-		if outboundLink.SuggestedProtocol == tap.TLSProtocol {
-			marshaledData, err := models.CreateWebsocketOutboundLinkMessage(outboundLink)
-			if err != nil {
-				rlog.Infof("Error converting outbound link to json %s, (%v,%+v)", err, err, err)
-				continue
-			}
-
-			err = connection.WriteMessage(websocket.TextMessage, marshaledData)
-			if err != nil {
-				rlog.Infof("error sending outbound link message through socket server %s, (%v,%+v)", err, err, err)
-				continue
-			}
-		}
-	}
-}

agent/pkg/api/main.go

@@ -7,7 +7,7 @@ import (
 	"fmt"
 	"mizuserver/pkg/database"
 	"mizuserver/pkg/holder"
-	"net/url"
+	"mizuserver/pkg/providers"
 	"os"
 	"path"
 	"sort"
@@ -18,7 +18,6 @@ import (
 
 	"github.com/google/martian/har"
 	"github.com/romana/rlog"
-	"github.com/up9inc/mizu/tap"
 	tapApi "github.com/up9inc/mizu/tap/api"
 
 	"mizuserver/pkg/models"
@@ -59,8 +58,10 @@ func StartReadingEntries(harChannel <-chan *tapApi.OutputChannelItem, workingDir
 }
 
 func startReadingFiles(workingDir string) {
-	err := os.MkdirAll(workingDir, os.ModePerm)
-	utils.CheckErr(err)
+	if err := os.MkdirAll(workingDir, os.ModePerm); err != nil {
+		rlog.Errorf("Failed to make dir: %s, err: %v", workingDir, err)
+		return
+	}
 
 	for true {
 		dir, _ := os.Open(workingDir)
@@ -88,17 +89,6 @@ func startReadingFiles(workingDir string) {
 		decErr := json.NewDecoder(bufio.NewReader(file)).Decode(&inputHar)
 		utils.CheckErr(decErr)
 
-		// for _, entry := range inputHar.Log.Entries {
-		// 	time.Sleep(time.Millisecond * 250)
-		// 	// connectionInfo := &tap.ConnectionInfo{
-		// 	// 	ClientIP:   fileInfo.Name(),
-		// 	// 	ClientPort: "",
-		// 	// 	ServerIP:   "",
-		// 	// 	ServerPort: "",
-		// 	// 	IsOutgoing: false,
-		// 	// }
-		// 	// saveHarToDb(entry, connectionInfo)
-		// }
 		rmErr := os.Remove(inputFilePath)
 		utils.CheckErr(rmErr)
 	}
@@ -110,6 +100,8 @@ func startReadingChannel(outputItems <-chan *tapApi.OutputChannelItem, extension
 	}
 
 	for item := range outputItems {
+		providers.EntryAdded()
+
 		extension := extensionsMap[item.Protocol.Name]
 		resolvedSource, resolvedDestionation := resolveIP(item.ConnectionInfo)
 		mizuEntry := extension.Dissector.Analyze(item, primitive.NewObjectID().Hex(), resolvedSource, resolvedDestionation)
@@ -123,7 +115,6 @@ func startReadingChannel(outputItems <-chan *tapApi.OutputChannelItem, extension
 			if err == nil {
 				rules, _ := models.RunValidationRulesState(*harEntry, mizuEntry.Service)
 				baseEntry.Rules = rules
-				baseEntry.Latency = mizuEntry.ElapsedTime
 			}
 		}
 
@@ -132,13 +123,6 @@ func startReadingChannel(outputItems <-chan *tapApi.OutputChannelItem, extension
 	}
 }
 
-func StartReadingOutbound(outboundLinkChannel <-chan *tap.OutboundLink) {
-	// tcpStreamFactory will block on write to channel. Empty channel to unblock.
-	// TODO: Make write to channel optional.
-	for range outboundLinkChannel {
-	}
-}
-
 func resolveIP(connectionInfo *tapApi.ConnectionInfo) (resolvedSource string, resolvedDestination string) {
 	if k8sResolver != nil {
 		unresolvedSource := connectionInfo.ClientIP
@@ -161,12 +145,6 @@ func resolveIP(connectionInfo *tapApi.ConnectionInfo) (resolvedSource string, re
 	return resolvedSource, resolvedDestination
 }
 
-func getServiceNameFromUrl(inputUrl string) (string, string) {
-	parsed, err := url.Parse(inputUrl)
-	utils.CheckErr(err)
-	return fmt.Sprintf("%s://%s", parsed.Scheme, parsed.Host), parsed.Path
-}
-
 func CheckIsServiceIP(address string) bool {
 	if k8sResolver == nil {
 		return false

agent/pkg/database/main.go

@@ -61,7 +61,7 @@ func GetEntriesFromDb(timestampFrom int64, timestampTo int64, protocolName *stri
 	order := OrderDesc
 	protocolNameCondition := "1 = 1"
 	if protocolName != nil {
-		protocolNameCondition = fmt.Sprintf("protocolKey = '%s'", *protocolName)
+		protocolNameCondition = fmt.Sprintf("protocolName = '%s'", *protocolName)
 	}
 
 	var entries []tapApi.MizuEntry

agent/pkg/rules/rulesHTTP.go

@@ -4,6 +4,7 @@ import (
 	"encoding/base64"
 	"encoding/json"
 	"fmt"
+	"github.com/romana/rlog"
 	"reflect"
 	"regexp"
 	"strings"
@@ -65,7 +66,7 @@ func MatchRequestPolicy(harEntry har.Entry, service string) []RulesMatched {
 				if err != nil {
 					continue
 				}
-				fmt.Println(matchValue, rule.Value)
+				rlog.Info(matchValue, rule.Value)
 			} else {
 				val := fmt.Sprint(out)
 				matchValue, err = regexp.MatchString(rule.Value, val)

agent/pkg/utils/har.go

@@ -4,13 +4,12 @@ import (
 	"bytes"
 	"errors"
 	"fmt"
-	"strconv"
-	"strings"
-	"time"
-
 	"github.com/google/martian/har"
 	"github.com/romana/rlog"
 	"github.com/up9inc/mizu/tap/api"
+	"strconv"
+	"strings"
+	"time"
 )
 
 // Keep it because we might want cookies in the future

agent/pkg/utils/socket_client.go

@@ -1,8 +1,8 @@
-package shared
+package utils
 
 import (
-	"fmt"
 	"github.com/gorilla/websocket"
+	"github.com/romana/rlog"
 	"time"
 )
 
@@ -11,24 +11,23 @@ const (
 	DEFAULT_SOCKET_RETRY_SLEEP_TIME = time.Second * 10
 )
 
-func ConnectToSocketServer(address string, retries int, retrySleepTime time.Duration, hideTimeoutErrors bool) (*websocket.Conn, error) {
+func ConnectToSocketServer(address string) (*websocket.Conn, error) {
 	var err error
 	var connection *websocket.Conn
 	try := 0
 
 	// Connection to server fails if client pod is up before server.
 	// Retries solve this issue.
-	for try < retries {
+	for try < DEFAULT_SOCKET_RETRIES {
+		rlog.Infof("Trying to connect to websocket: %s, attempt: %v/%v", address, try, DEFAULT_SOCKET_RETRIES)
 		connection, _, err = websocket.DefaultDialer.Dial(address, nil)
 		if err != nil {
+			rlog.Warnf("Failed connecting to websocket: %s, attempt: %v/%v, err: %s, (%v,%+v)", address, try, DEFAULT_SOCKET_RETRIES, err, err, err)
 			try++
-			if !hideTimeoutErrors {
-				fmt.Printf("Failed connecting to websocket server: %s, (%v,%+v)\n", err, err, err)
-			}
 		} else {
 			break
 		}
-		time.Sleep(retrySleepTime)
+		time.Sleep(DEFAULT_SOCKET_RETRY_SLEEP_TIME)
 	}
 
 	if err != nil {

agent/pkg/utils/utils.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"github.com/gin-gonic/gin"
 	"github.com/romana/rlog"
-	"log"
 	"net/http"
 	"net/url"
 	"os"
@@ -18,8 +17,8 @@ import (
 func StartServer(app *gin.Engine) {
 	signals := make(chan os.Signal, 2)
 	signal.Notify(signals,
-		os.Interrupt,  	  // this catch ctrl + c
-		syscall.SIGTSTP,  // this catch ctrl + z
+		os.Interrupt,    // this catch ctrl + c
+		syscall.SIGTSTP, // this catch ctrl + z
 	)
 
 	srv := &http.Server{
@@ -36,8 +35,9 @@ func StartServer(app *gin.Engine) {
 	}()
 
 	// Run server.
+	rlog.Infof("Starting the server...")
 	if err := app.Run(":8899"); err != nil {
-		log.Printf("Oops... Server is not running! Reason: %v", err)
+		rlog.Errorf("Server is not running! Reason: %v", err)
 	}
 }
 
@@ -54,15 +54,14 @@ func ReverseSlice(data interface{}) {
 
 func CheckErr(e error) {
 	if e != nil {
-		log.Printf("%v", e)
-		//panic(e)
+		rlog.Errorf("%v", e)
 	}
 }
 
 func SetHostname(address, newHostname string) string {
 	replacedUrl, err := url.Parse(address)
-	if err != nil{
-		log.Printf("error replacing hostname to %s in address %s, returning original %v",newHostname, address, err)
+	if err != nil {
+		rlog.Errorf("error replacing hostname to %s in address %s, returning original %v", newHostname, address, err)
 		return address
 	}
 	replacedUrl.Host = newHostname

cli/cmd/tap.go

@@ -67,7 +67,7 @@ func init() {
 	tapCmd.Flags().Bool(configStructs.DisableRedactionTapName, defaultTapConfig.DisableRedaction, "Disables redaction of potentially sensitive request/response headers and body values")
 	tapCmd.Flags().String(configStructs.HumanMaxEntriesDBSizeTapName, defaultTapConfig.HumanMaxEntriesDBSize, "Override the default max entries db size")
 	tapCmd.Flags().Bool(configStructs.DryRunTapName, defaultTapConfig.DryRun, "Preview of all pods matching the regex, without tapping them")
-	tapCmd.Flags().String(configStructs.EnforcePolicyFile, defaultTapConfig.EnforcePolicyFile, "Yaml file with policy rules")
+	tapCmd.Flags().String(configStructs.EnforcePolicyFile, defaultTapConfig.EnforcePolicyFile, "Yaml file path with policy rules")
 
 	tapCmd.Flags().String(configStructs.EnforcePolicyFileDeprecated, defaultTapConfig.EnforcePolicyFileDeprecated, "Yaml file with policy rules")
 	tapCmd.Flags().MarkDeprecated(configStructs.EnforcePolicyFileDeprecated, fmt.Sprintf("Use --%s instead", configStructs.EnforcePolicyFile))

cli/config/configStructs/tapConfig.go

@@ -16,7 +16,7 @@ const (
 	DisableRedactionTapName       = "no-redact"
 	HumanMaxEntriesDBSizeTapName  = "max-entries-db-size"
 	DryRunTapName                 = "dry-run"
-	EnforcePolicyFile             = "traffic-validation"
+	EnforcePolicyFile             = "traffic-validation-file"
 	EnforcePolicyFileDeprecated   = "test-rules"
 )
 
@@ -33,8 +33,8 @@ type TapConfig struct {
 	DisableRedaction             bool      `yaml:"no-redact" default:"false"`
 	HumanMaxEntriesDBSize        string    `yaml:"max-entries-db-size" default:"200MB"`
 	DryRun                       bool      `yaml:"dry-run" default:"false"`
-	EnforcePolicyFile            string    `yaml:"traffic-validation"`
-	EnforcePolicyFileDeprecated  string    `yaml:"test-rules"`
+	EnforcePolicyFile            string    `yaml:"traffic-validation-file"`
+	EnforcePolicyFileDeprecated  string    `yaml:"test-rules,omitempty" readonly:""`
 	ApiServerResources           Resources `yaml:"api-server-resources"`
 	TapperResources              Resources `yaml:"tapper-resources"`
 }

cli/mizu/controlSocket.go

@@ -1,42 +0,0 @@
-package mizu
-
-import (
-	"encoding/json"
-	"github.com/gorilla/websocket"
-	"github.com/up9inc/mizu/shared"
-	core "k8s.io/api/core/v1"
-	"time"
-)
-
-type ControlSocket struct {
-	connection *websocket.Conn
-}
-
-func CreateControlSocket(socketServerAddress string) (*ControlSocket, error) {
-	connection, err := shared.ConnectToSocketServer(socketServerAddress, 30, 2 * time.Second, true)
-	if err != nil {
-		return nil, err
-	} else {
-		return &ControlSocket{connection: connection}, nil
-	}
-}
-
-func (controlSocket *ControlSocket) SendNewTappedPodsListMessage(pods []core.Pod) error {
-	podInfos := make([]shared.PodInfo, 0)
-	for _, pod := range pods {
-		podInfos = append(podInfos, shared.PodInfo{Name: pod.Name, Namespace: pod.Namespace})
-	}
-	tapStatus := shared.TapStatus{Pods: podInfos}
-	socketMessage := shared.CreateWebSocketStatusMessage(tapStatus)
-
-	jsonMessage, err := json.Marshal(socketMessage)
-	if err != nil {
-		return err
-	}
-	err = controlSocket.connection.WriteMessage(websocket.TextMessage, jsonMessage)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}

docs/POLICY_RULES.md

@@ -1,34 +1,31 @@
 
-# API rules validation
+# Traffic validation rules
 
-This feature allows you to define set of simple rules, and test the API against them.
+This feature allows you to define set of simple rules, and test the traffic against them.
 Such validation may test response for specific JSON fields, headers, etc.
 
 ## Examples
 
-
-Example 1: HTTP request (REST API call) that didn’t pass validation is highlighted in red
+Example 1: HTTP request (REST API call) that didn't pass validation is highlighted in red
 
 ![Simple UI](../assets/validation-example1.png)
 
 - - -
 
-
 Example 2: Details pane shows the validation rule details and whether it passed or failed
 
 ![Simple UI](../assets/validation-example2.png)
 
 
 ## How to use
+
 To use this feature - create simple rules file (see details below) and pass this file as parameter to `mizu tap` command. For example, if rules are stored in file named `rules.yaml` — run the following command:
 
-
 ```shell
-mizu tap --traffic-validation rules.yaml PODNAME
+mizu tap --traffic-validation-file rules.yaml
 ```
 
 
-
 ## Rules file structure
 
 The structure of the traffic-validation-file is:
@@ -62,6 +59,7 @@ rules:
   service: "carts.*"
 ```
 
+
 ### Explanation:
 
 * First rule `holy-in-name-property`:
@@ -74,5 +72,4 @@ rules:
 
 * Third rule `latency-test`:
 
-  > This rule will be applied to all request made to `carts.*` services. If the latency of the response is greater than `1` will be marked as failure, marked as success otherwise.
-
+  > This rule will be applied to all request made to `carts.*` services. If the latency of the response is greater than `1ms` will be marked as failure, marked as success otherwise.

tap/api/api.go

@@ -156,7 +156,7 @@ type BaseEntryDetails struct {
 	SourcePort      string          `json:"sourcePort,omitempty"`
 	DestinationPort string          `json:"destinationPort,omitempty"`
 	IsOutgoing      bool            `json:"isOutgoing,omitempty"`
-	Latency         int64           `json:"latency,omitempty"`
+	Latency         int64           `json:"latency"`
 	Rules           ApplicableRules `json:"rules,omitempty"`
 }
 
@@ -190,6 +190,7 @@ func (bed *BaseEntryDetails) UnmarshalData(entry *MizuEntry) error {
 	bed.Timestamp = entry.Timestamp
 	bed.RequestSenderIp = entry.RequestSenderIp
 	bed.IsOutgoing = entry.IsOutgoing
+	bed.Latency = entry.ElapsedTime
 	return nil
 }
 

tap/extensions/amqp/main.go

@@ -312,7 +312,7 @@ func (d dissecting) Summarize(entry *api.MizuEntry) *api.BaseEntryDetails {
 		SourcePort:      entry.SourcePort,
 		DestinationPort: entry.DestinationPort,
 		IsOutgoing:      entry.IsOutgoing,
-		Latency:         0,
+		Latency:         entry.ElapsedTime,
 		Rules: api.ApplicableRules{
 			Latency: 0,
 			Status:  false,

tap/extensions/http/main.go

@@ -223,7 +223,7 @@ func (d dissecting) Summarize(entry *api.MizuEntry) *api.BaseEntryDetails {
 		SourcePort:      entry.SourcePort,
 		DestinationPort: entry.DestinationPort,
 		IsOutgoing:      entry.IsOutgoing,
-		Latency:         0,
+		Latency:         entry.ElapsedTime,
 		Rules: api.ApplicableRules{
 			Latency: 0,
 			Status:  false,

tap/extensions/http/sensitive_data_cleaner.go

@@ -86,6 +86,10 @@ func getContentTypeHeaderValue(headers http.Header) string {
 }
 
 func isFieldNameSensitive(fieldName string) bool {
+	if fieldName == ":authority" {
+		return false
+	}
+
 	name := strings.ToLower(fieldName)
 	name = strings.ReplaceAll(name, "_", "")
 	name = strings.ReplaceAll(name, "-", "")

tap/extensions/kafka/main.go

@@ -186,7 +186,7 @@ func (d dissecting) Summarize(entry *api.MizuEntry) *api.BaseEntryDetails {
 		SourcePort:      entry.SourcePort,
 		DestinationPort: entry.DestinationPort,
 		IsOutgoing:      entry.IsOutgoing,
-		Latency:         0,
+		Latency:         entry.ElapsedTime,
 		Rules: api.ApplicableRules{
 			Latency: 0,
 			Status:  false,

ui/package-lock.json

@@ -13454,9 +13454,9 @@
       }
     },
     "react-scrollable-feed-virtualized": {
-      "version": "1.4.2",
-      "resolved": "https://registry.npmjs.org/react-scrollable-feed-virtualized/-/react-scrollable-feed-virtualized-1.4.2.tgz",
-      "integrity": "sha512-j6M80ETqhSRBlygWe491gMPqCiAkVUQsLd/JR7DCKsZF44IYlJiunZWjdWe3//gxddTL68pjqq8pMvd1YN1bgw=="
+      "version": "1.4.3",
+      "resolved": "https://registry.npmjs.org/react-scrollable-feed-virtualized/-/react-scrollable-feed-virtualized-1.4.3.tgz",
+      "integrity": "sha512-M9WgJKr57jCyWKNCksc3oi+xhtO0YbL9d7Ll8Sdc5ZWOIstNvdNbNX0k4Nq6kXUVaHCJ9qE8omdSI/CxT3MLAQ=="
     },
     "react-syntax-highlighter": {
       "version": "15.4.3",

ui/package.json

@@ -21,7 +21,7 @@
     "react-copy-to-clipboard": "^5.0.3",
     "react-dom": "^17.0.2",
     "react-scripts": "4.0.3",
-    "react-scrollable-feed-virtualized": "^1.4.2",
+    "react-scrollable-feed-virtualized": "^1.4.3",
     "react-syntax-highlighter": "^15.4.3",
     "typescript": "^4.2.4",
     "web-vitals": "^1.1.1"