Fix some errors related to conversion to HAR (#286)

* Fix some errors related to conversion to HAR

* Fix the build error

* Fix the variable name

* Change to `Errorf`

README.md

@@ -143,7 +143,7 @@ Setting `mizu-resources-namespace=mizu` resets Mizu to its default behavior
 User-agent filtering (like health checks) - can be configured using command-line options:
 
 ```shell
-$ mizu tap "^ca.*" --set ignored-user-agents=kube-probe --set ignored-user-agents=prometheus
+$ mizu tap "^ca.*" --set tap.ignored-user-agents=kube-probe --set tap.ignored-user-agents=prometheus
 +carts-66c77f5fbb-fq65r
 +catalogue-5f4cb7cf5-7zrmn
 Web interface is now available at http://localhost:8899

acceptanceTests/logs_test.go

@@ -0,0 +1,196 @@
+package acceptanceTests
+
+import (
+	"archive/zip"
+	"os/exec"
+	"testing"
+)
+
+func TestLogs(t *testing.T) {
+	if testing.Short() {
+		t.Skip("ignored acceptance test")
+	}
+
+	cliPath, cliPathErr := getCliPath()
+	if cliPathErr != nil {
+		t.Errorf("failed to get cli path, err: %v", cliPathErr)
+		return
+	}
+
+	tapCmdArgs := getDefaultTapCommandArgs()
+
+	tapNamespace := getDefaultTapNamespace()
+	tapCmdArgs = append(tapCmdArgs, tapNamespace...)
+
+	tapCmd := exec.Command(cliPath, tapCmdArgs...)
+	t.Logf("running command: %v", tapCmd.String())
+
+	t.Cleanup(func() {
+		if err := cleanupCommand(tapCmd); err != nil {
+			t.Logf("failed to cleanup tap command, err: %v", err)
+		}
+	})
+
+	if err := tapCmd.Start(); err != nil {
+		t.Errorf("failed to start tap command, err: %v", err)
+		return
+	}
+
+	apiServerUrl := getApiServerUrl(defaultApiServerPort)
+
+	if err := waitTapPodsReady(apiServerUrl); err != nil {
+		t.Errorf("failed to start tap pods on time, err: %v", err)
+		return
+	}
+
+	logsCmdArgs := getDefaultLogsCommandArgs()
+
+	logsCmd := exec.Command(cliPath, logsCmdArgs...)
+	t.Logf("running command: %v", logsCmd.String())
+
+	if err := logsCmd.Start(); err != nil {
+		t.Errorf("failed to start logs command, err: %v", err)
+		return
+	}
+
+	if err := logsCmd.Wait(); err != nil {
+		t.Errorf("failed to wait logs command, err: %v", err)
+		return
+	}
+
+	logsPath, logsPathErr := getLogsPath()
+	if logsPathErr != nil {
+		t.Errorf("failed to get logs path, err: %v", logsPathErr)
+		return
+	}
+
+	zipReader, zipError := zip.OpenReader(logsPath)
+	if zipError != nil {
+		t.Errorf("failed to get zip reader, err: %v", zipError)
+		return
+	}
+
+	t.Cleanup(func() {
+		if err := zipReader.Close(); err != nil {
+			t.Logf("failed to close zip reader, err: %v", err)
+		}
+	})
+
+	var logsFileNames []string
+	for _, file := range zipReader.File {
+		logsFileNames = append(logsFileNames, file.Name)
+	}
+
+	if !Contains(logsFileNames, "mizu.mizu-api-server.log") {
+		t.Errorf("api server logs not found")
+		return
+	}
+
+	if !Contains(logsFileNames, "mizu_cli.log") {
+		t.Errorf("cli logs not found")
+		return
+	}
+
+	if !Contains(logsFileNames, "mizu_events.log") {
+		t.Errorf("events logs not found")
+		return
+	}
+
+	if !ContainsPartOfValue(logsFileNames, "mizu.mizu-tapper-daemon-set") {
+		t.Errorf("tapper logs not found")
+		return
+	}
+}
+
+func TestLogsPath(t *testing.T) {
+	if testing.Short() {
+		t.Skip("ignored acceptance test")
+	}
+
+	cliPath, cliPathErr := getCliPath()
+	if cliPathErr != nil {
+		t.Errorf("failed to get cli path, err: %v", cliPathErr)
+		return
+	}
+
+	tapCmdArgs := getDefaultTapCommandArgs()
+
+	tapNamespace := getDefaultTapNamespace()
+	tapCmdArgs = append(tapCmdArgs, tapNamespace...)
+
+	tapCmd := exec.Command(cliPath, tapCmdArgs...)
+	t.Logf("running command: %v", tapCmd.String())
+
+	t.Cleanup(func() {
+		if err := cleanupCommand(tapCmd); err != nil {
+			t.Logf("failed to cleanup tap command, err: %v", err)
+		}
+	})
+
+	if err := tapCmd.Start(); err != nil {
+		t.Errorf("failed to start tap command, err: %v", err)
+		return
+	}
+
+	apiServerUrl := getApiServerUrl(defaultApiServerPort)
+
+	if err := waitTapPodsReady(apiServerUrl); err != nil {
+		t.Errorf("failed to start tap pods on time, err: %v", err)
+		return
+	}
+
+	logsCmdArgs := getDefaultLogsCommandArgs()
+
+	logsPath := "../logs.zip"
+	logsCmdArgs = append(logsCmdArgs, "-f", logsPath)
+
+	logsCmd := exec.Command(cliPath, logsCmdArgs...)
+	t.Logf("running command: %v", logsCmd.String())
+
+	if err := logsCmd.Start(); err != nil {
+		t.Errorf("failed to start logs command, err: %v", err)
+		return
+	}
+
+	if err := logsCmd.Wait(); err != nil {
+		t.Errorf("failed to wait logs command, err: %v", err)
+		return
+	}
+
+	zipReader, zipError := zip.OpenReader(logsPath)
+	if zipError != nil {
+		t.Errorf("failed to get zip reader, err: %v", zipError)
+		return
+	}
+
+	t.Cleanup(func() {
+		if err := zipReader.Close(); err != nil {
+			t.Logf("failed to close zip reader, err: %v", err)
+		}
+	})
+
+	var logsFileNames []string
+	for _, file := range zipReader.File {
+		logsFileNames = append(logsFileNames, file.Name)
+	}
+
+	if !Contains(logsFileNames, "mizu.mizu-api-server.log") {
+		t.Errorf("api server logs not found")
+		return
+	}
+
+	if !Contains(logsFileNames, "mizu_cli.log") {
+		t.Errorf("cli logs not found")
+		return
+	}
+
+	if !Contains(logsFileNames, "mizu_events.log") {
+		t.Errorf("events logs not found")
+		return
+	}
+
+	if !ContainsPartOfValue(logsFileNames, "mizu.mizu-tapper-daemon-set") {
+		t.Errorf("tapper logs not found")
+		return
+	}
+}

acceptanceTests/tap_test.go

@@ -1,11 +1,14 @@
 package acceptanceTests
 
 import (
+	"archive/zip"
 	"bytes"
 	"encoding/json"
 	"fmt"
+	"io/ioutil"
 	"net/http"
 	"os/exec"
+	"path"
 	"strings"
 	"testing"
 	"time"
@@ -758,3 +761,105 @@ func TestTapRegexMasking(t *testing.T) {
 		return
 	}
 }
+
+func TestTapDumpLogs(t *testing.T) {
+	if testing.Short() {
+		t.Skip("ignored acceptance test")
+	}
+
+	cliPath, cliPathErr := getCliPath()
+	if cliPathErr != nil {
+		t.Errorf("failed to get cli path, err: %v", cliPathErr)
+		return
+	}
+
+	tapCmdArgs := getDefaultTapCommandArgs()
+
+	tapNamespace := getDefaultTapNamespace()
+	tapCmdArgs = append(tapCmdArgs, tapNamespace...)
+
+	tapCmdArgs = append(tapCmdArgs, "--set", "dump-logs=true")
+
+	tapCmd := exec.Command(cliPath, tapCmdArgs...)
+	t.Logf("running command: %v", tapCmd.String())
+
+	if err := tapCmd.Start(); err != nil {
+		t.Errorf("failed to start tap command, err: %v", err)
+		return
+	}
+
+	apiServerUrl := getApiServerUrl(defaultApiServerPort)
+
+	if err := waitTapPodsReady(apiServerUrl); err != nil {
+		t.Errorf("failed to start tap pods on time, err: %v", err)
+		return
+	}
+
+	if err := cleanupCommand(tapCmd); err != nil {
+		t.Errorf("failed to cleanup tap command, err: %v", err)
+		return
+	}
+
+	mizuFolderPath, mizuPathErr := getMizuFolderPath()
+	if mizuPathErr != nil {
+		t.Errorf("failed to get mizu folder path, err: %v", mizuPathErr)
+		return
+	}
+
+	files, readErr := ioutil.ReadDir(mizuFolderPath)
+	if readErr != nil {
+		t.Errorf("failed to read mizu folder files, err: %v", readErr)
+		return
+	}
+
+	var dumpsLogsPath string
+	for _, file := range files {
+		fileName := file.Name()
+		if strings.Contains(fileName, "mizu_logs") {
+			dumpsLogsPath = path.Join(mizuFolderPath, fileName)
+			break
+		}
+	}
+
+	if dumpsLogsPath == "" {
+		t.Errorf("dump logs file not found")
+		return
+	}
+
+	zipReader, zipError := zip.OpenReader(dumpsLogsPath)
+	if zipError != nil {
+		t.Errorf("failed to get zip reader, err: %v", zipError)
+		return
+	}
+
+	t.Cleanup(func() {
+		if err := zipReader.Close(); err != nil {
+			t.Logf("failed to close zip reader, err: %v", err)
+		}
+	})
+
+	var logsFileNames []string
+	for _, file := range zipReader.File {
+		logsFileNames = append(logsFileNames, file.Name)
+	}
+
+	if !Contains(logsFileNames, "mizu.mizu-api-server.log") {
+		t.Errorf("api server logs not found")
+		return
+	}
+
+	if !Contains(logsFileNames, "mizu_cli.log") {
+		t.Errorf("cli logs not found")
+		return
+	}
+
+	if !Contains(logsFileNames, "mizu_events.log") {
+		t.Errorf("events logs not found")
+		return
+	}
+
+	if !ContainsPartOfValue(logsFileNames, "mizu.mizu-tapper-daemon-set") {
+		t.Errorf("tapper logs not found")
+		return
+	}
+}

acceptanceTests/testsUtils.go

@@ -9,6 +9,7 @@ import (
 	"os"
 	"os/exec"
 	"path"
+	"strings"
 	"syscall"
 	"time"
 )
@@ -32,13 +33,22 @@ func getCliPath() (string, error) {
 	return cliPath, nil
 }
 
-func getConfigPath() (string, error) {
+func getMizuFolderPath() (string, error) {
 	home, homeDirErr := os.UserHomeDir()
 	if homeDirErr != nil {
 		return "", homeDirErr
 	}
 
-	return path.Join(home, ".mizu", "config.yaml"), nil
+	return path.Join(home, ".mizu"), nil
+}
+
+func getConfigPath() (string, error) {
+	mizuFolderPath, mizuPathError := getMizuFolderPath()
+	if mizuPathError != nil {
+		return "", mizuPathError
+	}
+
+	return path.Join(mizuFolderPath, "config.yaml"), nil
 }
 
 func getProxyUrl(namespace string, service string) string {
@@ -72,6 +82,13 @@ func getDefaultTapCommandArgsWithRegex(regex string) []string {
 	return append([]string{tapCommand, regex}, defaultCmdArgs...)
 }
 
+func getDefaultLogsCommandArgs() []string {
+	logsCommand := "logs"
+	defaultCmdArgs := getDefaultCommandArgs()
+
+	return append([]string{logsCommand}, defaultCmdArgs...)
+}
+
 func getDefaultTapNamespace() []string {
 	return []string{"-n", "mizu-tests"}
 }
@@ -193,3 +210,33 @@ func getPods(tapStatusInterface interface{}) ([]map[string]interface{}, error) {
 
 	return pods, nil
 }
+
+func getLogsPath() (string, error) {
+	dir, filePathErr := os.Getwd()
+	if filePathErr != nil {
+		return "", filePathErr
+	}
+
+	logsPath := path.Join(dir, "mizu_logs.zip")
+	return logsPath, nil
+}
+
+func Contains(slice []string, containsValue string) bool {
+	for _, sliceValue := range slice {
+		if sliceValue == containsValue {
+			return true
+		}
+	}
+
+	return false
+}
+
+func ContainsPartOfValue(slice []string, containsValue string) bool {
+	for _, sliceValue := range slice {
+		if strings.Contains(sliceValue, containsValue) {
+			return true
+		}
+	}
+
+	return false
+}

agent/pkg/api/main.go

@@ -119,10 +119,12 @@ func startReadingChannel(outputItems <-chan *tapApi.OutputChannelItem, extension
 		if extension.Protocol.Name == "http" {
 			var pair tapApi.RequestResponsePair
 			json.Unmarshal([]byte(mizuEntry.Entry), &pair)
-			harEntry, _ := utils.NewEntry(&pair)
-			rules, _ := models.RunValidationRulesState(*harEntry, mizuEntry.Service)
-			baseEntry.Rules = rules
-			baseEntry.Latency = mizuEntry.ElapsedTime
+			harEntry, err := utils.NewEntry(&pair)
+			if err == nil {
+				rules, _ := models.RunValidationRulesState(*harEntry, mizuEntry.Service)
+				baseEntry.Rules = rules
+				baseEntry.Latency = mizuEntry.ElapsedTime
+			}
 		}
 
 		baseEntryBytes, _ := models.CreateBaseEntryWebSocketMessage(baseEntry)

agent/pkg/utils/har.go

@@ -9,11 +9,10 @@ import (
 	"time"
 
 	"github.com/google/martian/har"
-	"github.com/up9inc/mizu/tap"
+	"github.com/romana/rlog"
 	"github.com/up9inc/mizu/tap/api"
 )
 
-
 // Keep it because we might want cookies in the future
 //func BuildCookies(rawCookies []interface{}) []har.Cookie {
 //	cookies := make([]har.Cookie, 0, len(rawCookies))
@@ -82,7 +81,7 @@ func BuildHeaders(rawHeaders []interface{}) ([]har.Header, string, string, strin
 			path = h["value"].(string)
 		}
 		if h["name"] == ":status" {
-			path = h["value"].(string)
+			status = h["value"].(string)
 		}
 	}
 
@@ -205,7 +204,7 @@ func NewResponse(response *api.GenericMessage) (harResponse *har.Response, err e
 	if strings.HasPrefix(mimeType.(string), "application/grpc") {
 		status, err = strconv.Atoi(_status)
 		if err != nil {
-			tap.SilentError("convert-response-status-for-har", "Failed converting status to int %s (%v,%+v)", err, err, err)
+			rlog.Errorf("Failed converting status to int %s (%v,%+v)", err, err, err)
 			return nil, errors.New("failed converting response status to int for HAR")
 		}
 	}
@@ -226,14 +225,13 @@ func NewResponse(response *api.GenericMessage) (harResponse *har.Response, err e
 func NewEntry(pair *api.RequestResponsePair) (*har.Entry, error) {
 	harRequest, err := NewRequest(&pair.Request)
 	if err != nil {
-		tap.SilentError("convert-request-to-har", "Failed converting request to HAR %s (%v,%+v)", err, err, err)
+		rlog.Errorf("Failed converting request to HAR %s (%v,%+v)", err, err, err)
 		return nil, errors.New("failed converting request to HAR")
 	}
 
 	harResponse, err := NewResponse(&pair.Response)
 	if err != nil {
-		fmt.Printf("err: %+v\n", err)
-		tap.SilentError("convert-response-to-har", "Failed converting response to HAR %s (%v,%+v)", err, err, err)
+		rlog.Errorf("Failed converting response to HAR %s (%v,%+v)", err, err, err)
 		return nil, errors.New("failed converting response to HAR")
 	}