Adding (basic) configuration (#135)

This commit is contained in:
gadotroee
2021-07-26 11:23:35 +03:00
committed by GitHub
parent 6dd2bf705b
commit f175480f65
15 changed files with 340 additions and 32 deletions

34
cli/cmd/config.go Normal file
View File

@@ -0,0 +1,34 @@
package cmd
import (
"fmt"
"github.com/spf13/cobra"
"github.com/up9inc/mizu/cli/mizu"
"github.com/up9inc/mizu/cli/uiUtils"
"os"
)
var outputFileName string
var configCmd = &cobra.Command{
Use: "config",
Short: "Generate example config file to stdout",
RunE: func(cmd *cobra.Command, args []string) error {
template := mizu.GetTemplateConfig()
if outputFileName != "" {
data := []byte(template)
_ = os.WriteFile(outputFileName, data, 0644)
mizu.Log.Infof(fmt.Sprintf("Template File written to %s", fmt.Sprintf(uiUtils.Purple, outputFileName)))
} else {
mizu.Log.Debugf("Writing template config.\n%v", template)
fmt.Printf("%v", template)
}
return nil
},
}
func init() {
rootCmd.AddCommand(configCmd)
configCmd.Flags().StringVarP(&outputFileName, "file", "f", "", "Save content to local file")
}

View File

@@ -1,14 +1,32 @@
package cmd
import (
"errors"
"fmt"
"github.com/spf13/cobra"
"github.com/up9inc/mizu/cli/mizu"
)
var commandLineFlags []string
var rootCmd = &cobra.Command{
Use: "mizu",
Short: "A web traffic viewer for kubernetes",
Long: `A web traffic viewer for kubernetes
Further info is available at https://github.com/up9inc/mizu`,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if err := mizu.InitConfig(commandLineFlags); err != nil {
mizu.Log.Errorf("Invalid config, Exit %s", err)
return errors.New(fmt.Sprintf("%v", err))
}
prettifiedConfig := mizu.GetConfigStr()
mizu.Log.Debugf("Final Config: %s", prettifiedConfig)
return nil
},
}
func init() {
rootCmd.PersistentFlags().StringSliceVar(&commandLineFlags, "set", []string{}, "Override values using --set")
}
// Execute adds all child commands to the root command and sets flags appropriately.

View File

@@ -36,7 +36,7 @@ var regex *regexp.Regexp
const maxEntriesDBSizeFlagName = "max-entries-db-size"
const analysisMessageToConfirm = `NOTE: running mizu with --analysis flag will upload recorded traffic
to UP9 cloud for further analysis and enriched presentation options.
for further analysis and enriched presentation options.
`
var tapCmd = &cobra.Command{
@@ -48,8 +48,16 @@ Supported protocols are HTTP and gRPC.`,
go mizu.ReportRun("tap", mizuTapOptions)
RunMizuTap(regex, mizuTapOptions)
return nil
},
PreRunE: func(cmd *cobra.Command, args []string) error {
mizu.Log.Info("Getting params")
mizuTapOptions.AnalysisDestination = mizu.GetString(mizu.ConfigurationKeyAnalyzingDestination)
mizuTapOptions.SleepIntervalSec = uint16(mizu.GetInt(mizu.ConfigurationKeyUploadInterval))
mizuTapOptions.MizuImage = mizu.GetString(mizu.ConfigurationKeyMizuImage)
mizu.Log.Infof(uiUtils.PrettyJson(mizuTapOptions))
if len(args) == 0 {
return errors.New("POD REGEX argument is required")
} else if len(args) > 1 {
@@ -95,11 +103,8 @@ func init() {
tapCmd.Flags().Uint16VarP(&mizuTapOptions.GuiPort, "gui-port", "p", 8899, "Provide a custom port for the web interface webserver")
tapCmd.Flags().StringVarP(&mizuTapOptions.Namespace, "namespace", "n", "", "Namespace selector")
tapCmd.Flags().BoolVar(&mizuTapOptions.Analysis, "analysis", false, "Uploads traffic to UP9 for further analysis (Beta)")
tapCmd.Flags().StringVar(&mizuTapOptions.AnalysisDestination, "dest", "up9.app", "Destination environment")
tapCmd.Flags().Uint16VarP(&mizuTapOptions.SleepIntervalSec, "upload-interval", "", 10, "Interval in seconds for uploading data to UP9")
tapCmd.Flags().BoolVarP(&mizuTapOptions.AllNamespaces, "all-namespaces", "A", false, "Tap all namespaces")
tapCmd.Flags().StringVarP(&mizuTapOptions.KubeConfigPath, "kube-config", "k", "", "Path to kube-config file")
tapCmd.Flags().StringVarP(&mizuTapOptions.MizuImage, "mizu-image", "", fmt.Sprintf("gcr.io/up9-docker-hub/mizu/%s:%s", mizu.Branch, mizu.SemVer), "Custom image for mizu API server")
tapCmd.Flags().StringArrayVarP(&mizuTapOptions.PlainTextFilterRegexes, "regex-masking", "r", nil, "List of regex expressions that are used to filter matching values from text/plain http bodies")
tapCmd.Flags().StringVarP(&direction, "direction", "", "in", "Record traffic that goes in this direction (relative to the tapped pod): in/any")
tapCmd.Flags().BoolVar(&mizuTapOptions.HideHealthChecks, "hide-healthchecks", false, "hides requests with kube-probe or prometheus user-agent headers")

View File

@@ -5,6 +5,7 @@ import (
"fmt"
"github.com/up9inc/mizu/cli/kubernetes"
"github.com/up9inc/mizu/cli/mizu"
"github.com/up9inc/mizu/cli/uiUtils"
"github.com/up9inc/mizu/shared"
"github.com/up9inc/mizu/shared/debounce"
core "k8s.io/api/core/v1"
@@ -39,11 +40,11 @@ func RunMizuTap(podRegexQuery *regexp.Regexp, tappingOptions *MizuTapOptions) {
kubernetesProvider, err := kubernetes.NewProvider(tappingOptions.KubeConfigPath)
if err != nil {
if clientcmd.IsEmptyConfig(err) {
mizu.Log.Infof(mizu.Red, "Couldn't find the kube config file, or file is empty. Try adding '--kube-config=<path to kube config file>'\n")
mizu.Log.Infof(uiUtils.Red, "Couldn't find the kube config file, or file is empty. Try adding '--kube-config=<path to kube config file>'\n")
return
}
if clientcmd.IsConfigurationInvalid(err) {
mizu.Log.Infof(mizu.Red, "Invalid kube config file. Try using a different config with '--kube-config=<path to kube config file>'\n")
mizu.Log.Infof(uiUtils.Red, "Invalid kube config file. Try using a different config with '--kube-config=<path to kube config file>'\n")
return
}
}
@@ -259,10 +260,10 @@ func watchPodsForTapping(ctx context.Context, kubernetesProvider *kubernetes.Pro
for {
select {
case newTarget := <-added:
mizu.Log.Infof(mizu.Green, fmt.Sprintf("+%s\n", newTarget.Name))
mizu.Log.Infof(uiUtils.Green, fmt.Sprintf("+%s\n", newTarget.Name))
case removedTarget := <-removed:
mizu.Log.Infof(mizu.Red, fmt.Sprintf("-%s\n", removedTarget.Name))
mizu.Log.Infof(uiUtils.Red, fmt.Sprintf("-%s\n", removedTarget.Name))
restartTappersDebouncer.SetOn()
case modifiedTarget := <-modified:
@@ -327,7 +328,7 @@ func portForwardApiPod(ctx context.Context, kubernetesProvider *kubernetes.Provi
if response, err := http.Get(u.String()); err != nil || response.StatusCode != 200 {
mizu.Log.Infof("error sending upload entries req, status code: %v, err: %v\n", response.StatusCode, err)
} else {
mizu.Log.Infof(mizu.Purple, "Traffic is uploading to UP9 for further analysis\n")
mizu.Log.Infof(uiUtils.Purple, "Traffic is uploading to UP9 for further analysis\n")
}
}
}

View File

@@ -5,6 +5,7 @@ import (
"fmt"
"github.com/up9inc/mizu/cli/kubernetes"
"github.com/up9inc/mizu/cli/mizu"
"github.com/up9inc/mizu/cli/uiUtils"
"k8s.io/client-go/tools/clientcmd"
"net/http"
)
@@ -17,7 +18,7 @@ func runMizuView(mizuViewOptions *MizuViewOptions) {
return
}
if clientcmd.IsConfigurationInvalid(err) {
mizu.Log.Infof(mizu.Red, "Invalid kube config file. Try using a different config with '--kube-config=<path to kube config file>'\n")
mizu.Log.Infof(uiUtils.Red, "Invalid kube config file. Try using a different config with '--kube-config=<path to kube config file>'\n")
return
}
}

View File

@@ -8,6 +8,7 @@ require (
github.com/op/go-logging v0.0.0-20160315200505-970db520ece7
github.com/spf13/cobra v1.1.3
github.com/up9inc/mizu/shared v0.0.0
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
k8s.io/api v0.21.2
k8s.io/apimachinery v0.21.2
k8s.io/client-go v0.21.2

View File

@@ -367,8 +367,6 @@ github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4O
github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/romana/rlog v0.0.0-20171115192701-f018bc92e7d7 h1:jkvpcEatpwuMF5O5LVxTnehj6YZ/aEZN4NWD/Xml4pI=
github.com/romana/rlog v0.0.0-20171115192701-f018bc92e7d7/go.mod h1:KTrHyWpO1sevuXPZwyeZc72ddWRFqNSKDFl7uVWKpg0=
github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
@@ -691,8 +689,9 @@ gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=
gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

211
cli/mizu/config.go Normal file
View File

@@ -0,0 +1,211 @@
package mizu
import (
"errors"
"fmt"
"github.com/up9inc/mizu/cli/uiUtils"
"gopkg.in/yaml.v3"
"io/ioutil"
"os"
"path"
"reflect"
"strconv"
"strings"
)
const separator = "="
var configObj = map[string]interface{}{}
type CommandLineFlag struct {
CommandLineName string
YamlHierarchyName string
DefaultValue interface{}
Type reflect.Kind
}
const (
ConfigurationKeyAnalyzingDestination = "tap.dest"
ConfigurationKeyUploadInterval = "tap.uploadInterval"
ConfigurationKeyMizuImage = "mizuImage"
)
var allowedSetFlags = []CommandLineFlag{
{
CommandLineName: "dest",
YamlHierarchyName: ConfigurationKeyAnalyzingDestination,
DefaultValue: "up9.app",
Type: reflect.String,
// TODO: maybe add short description that we can show
},
{
CommandLineName: "uploadInterval",
YamlHierarchyName: ConfigurationKeyUploadInterval,
DefaultValue: 10,
Type: reflect.Int,
},
{
CommandLineName: "mizuImage",
YamlHierarchyName: ConfigurationKeyMizuImage,
DefaultValue: fmt.Sprintf("gcr.io/up9-docker-hub/mizu/%s:%s", Branch, SemVer),
Type: reflect.String,
},
}
func GetString(key string) string {
return fmt.Sprintf("%v", getValueFromMergedConfig(key))
}
func GetInt(key string) int {
stringVal := GetString(key)
Log.Debugf("Found string value %v", stringVal)
val, err := strconv.Atoi(stringVal)
if err != nil {
Log.Warningf("Invalid value %v for key %s", val, key)
os.Exit(1)
}
return val
}
func InitConfig(commandLineValues []string) error {
Log.Debugf("Merging default values")
mergeDefaultValues()
Log.Debugf("Merging config file values")
if err1 := mergeConfigFile(); err1 != nil {
Log.Infof(fmt.Sprintf(uiUtils.Red, "Invalid config file\n"))
return err1
}
Log.Debugf("Merging command line values")
if err2 := mergeCommandLineFlags(commandLineValues); err2 != nil {
Log.Infof(fmt.Sprintf(uiUtils.Red, "Invalid commanad argument\n"))
return err2
}
finalConfigPrettified, _ := uiUtils.PrettyJson(configObj)
Log.Debugf("Merged all config successfully\n Final config: %v", finalConfigPrettified)
return nil
}
func GetTemplateConfig() string {
templateConfig := map[string]interface{}{}
for _, allowedFlag := range allowedSetFlags {
addToConfigObj(allowedFlag.YamlHierarchyName, allowedFlag.DefaultValue, templateConfig)
}
prettifiedConfig, _ := uiUtils.PrettyYaml(templateConfig)
return prettifiedConfig
}
func GetConfigStr() string {
val, _ := uiUtils.PrettyYaml(configObj)
return val
}
func getValueFromMergedConfig(key string) interface{} {
if a, ok := configObj[key]; ok {
return a
}
return nil
}
func mergeDefaultValues() {
for _, allowedFlag := range allowedSetFlags {
Log.Debugf("Setting %v to %v", allowedFlag.YamlHierarchyName, allowedFlag.DefaultValue)
configObj[allowedFlag.YamlHierarchyName] = allowedFlag.DefaultValue
}
}
func mergeConfigFile() error {
Log.Debugf("Merging mizu config file values")
home, homeDirErr := os.UserHomeDir()
if homeDirErr != nil {
return nil
}
reader, openErr := os.Open(path.Join(home, ".mizu", "config.yaml"))
if openErr != nil {
return nil
}
buf, readErr := ioutil.ReadAll(reader)
if readErr != nil {
return readErr
}
m := make(map[string]interface{})
if err := yaml.Unmarshal(buf, &m); err != nil {
return err
}
for k, v := range m {
addToConfig(k, v)
}
return nil
}
func addToConfig(prefix string, value interface{}) {
typ := reflect.TypeOf(value).Kind()
if typ == reflect.Int || typ == reflect.String || typ == reflect.Slice {
validateConfigFileKey(prefix)
configObj[prefix] = value
} else if typ == reflect.Map {
for k1, v1 := range value.(map[string]interface{}) {
addToConfig(fmt.Sprintf("%s.%s", prefix, k1), v1)
}
}
}
func mergeCommandLineFlags(commandLineValues []string) error {
Log.Debugf("Merging Command line flags")
for _, e := range commandLineValues {
if !strings.Contains(e, separator) {
return errors.New(fmt.Sprintf("invalid set argument %s", e))
}
split := strings.SplitN(e, separator, 2)
if len(split) != 2 {
return errors.New(fmt.Sprintf("invalid set argument %s", e))
}
setFlagKey, argumentValue := split[0], split[1]
argumentNameInConfig, expectedType, err := flagFromAllowed(setFlagKey)
if err != nil {
return err
}
argumentType := reflect.ValueOf(argumentValue).Kind()
if argumentType != expectedType {
return errors.New(fmt.Sprintf("Invalid value for argument %s (should be type %s but got %s", setFlagKey, expectedType, argumentType))
}
configObj[argumentNameInConfig] = argumentValue
}
return nil
}
func flagFromAllowed(setFlagKey string) (string, reflect.Kind, error) {
for _, allowedFlag := range allowedSetFlags {
if strings.ToLower(allowedFlag.CommandLineName) == strings.ToLower(setFlagKey) {
return allowedFlag.YamlHierarchyName, allowedFlag.Type, nil
}
}
return "", reflect.Invalid, errors.New(fmt.Sprintf("invalid set argument %s", setFlagKey))
}
func validateConfigFileKey(configFileKey string) {
for _, allowedFlag := range allowedSetFlags {
if allowedFlag.YamlHierarchyName == configFileKey {
return
}
}
Log.Info(fmt.Sprintf("Unknown argument: %s. Exit", configFileKey))
os.Exit(1)
}
func addToConfigObj(key string, value interface{}, configObj map[string]interface{}) {
typ := reflect.TypeOf(value).Kind()
if typ == reflect.Int || typ == reflect.String || typ == reflect.Slice {
if strings.Contains(key, ".") {
split := strings.SplitN(key, ".", 2)
firstLevelKey := split[0]
if _, ok := configObj[firstLevelKey]; !ok {
configObj[firstLevelKey] = map[string]interface{}{}
}
addToConfigObj(split[1], value, configObj[firstLevelKey].(map[string]interface{}))
} else {
configObj[key] = value
}
}
}

View File

@@ -18,14 +18,3 @@ const (
TapperDaemonSetName = "mizu-tapper-daemon-set"
TapperPodName = "mizu-tapper"
)
const (
Black = "\033[1;30m%s\033[0m"
Red = "\033[1;31m%s\033[0m"
Green = "\033[1;32m%s\033[0m"
Yellow = "\033[1;33m%s\033[0m"
Purple = "\033[1;34m%s\033[0m"
Magenta = "\033[1;35m%s\033[0m"
Teal = "\033[1;36m%s\033[0m"
White = "\033[1;37m%s\033[0m"
)

View File

@@ -14,7 +14,7 @@ var format = logging.MustStringFormatter(
)
func InitLogger() {
homeDirPath, err := os.UserHomeDir()
homeDirPath, _ := os.UserHomeDir()
mizuDirPath := path.Join(homeDirPath, ".mizu")
if err := os.MkdirAll(mizuDirPath, os.ModePerm); err != nil {
panic(fmt.Sprintf("Failed creating .mizu dir: %v, err %v", mizuDirPath, err))

View File

@@ -26,8 +26,7 @@ func ReportRun(cmd string, args interface{}) {
jsonValue, _ := json.Marshal(argsMap)
if resp, err := http.Post(telemetryUrl,
"application/json", bytes.NewBuffer(jsonValue)); err != nil {
if resp, err := http.Post(telemetryUrl, "application/json", bytes.NewBuffer(jsonValue)); err != nil {
Log.Debugf("error sending telemetry err: %v, response %v", err, resp)
} else {
Log.Debugf("Successfully reported telemetry")

View File

@@ -5,6 +5,7 @@ import (
"encoding/json"
"fmt"
"github.com/google/go-github/v37/github"
"github.com/up9inc/mizu/cli/uiUtils"
"github.com/up9inc/mizu/shared"
"github.com/up9inc/mizu/shared/semver"
"io/ioutil"
@@ -44,7 +45,7 @@ func CheckVersionCompatibility(port uint16) (bool, error) {
return true, nil
}
Log.Infof(Red, fmt.Sprintf("cli version (%s) is not compatible with api version (%s)\n", SemVer, apiSemVer))
Log.Infof(uiUtils.Red, fmt.Sprintf("cli version (%s) is not compatible with api version (%s)\n", SemVer, apiSemVer))
return false, nil
}
@@ -86,6 +87,6 @@ func CheckNewerVersion() {
gitHubVersion = gitHubVersion[:len(gitHubVersion)-1]
Log.Debugf("Finished version validation, took %v", time.Since(start))
if SemVer < gitHubVersion {
Log.Infof(Yellow, fmt.Sprintf("Update available! %v -> %v (%v)\n", SemVer, gitHubVersion, *latestRelease.HTMLURL))
Log.Infof(uiUtils.Yellow, fmt.Sprintf("Update available! %v -> %v (%v)\n", SemVer, gitHubVersion, *latestRelease.HTMLURL))
}
}

13
cli/uiUtils/colors.go Normal file
View File

@@ -0,0 +1,13 @@
package uiUtils
const (
Black = "\033[1;30m%s\033[0m"
Red = "\033[1;31m%s\033[0m"
Green = "\033[1;32m%s\033[0m"
Yellow = "\033[1;33m%s\033[0m"
Purple = "\033[1;34m%s\033[0m"
Magenta = "\033[1;35m%s\033[0m"
Teal = "\033[1;36m%s\033[0m"
White = "\033[1;37m%s\033[0m"
)

View File

@@ -2,7 +2,7 @@ package uiUtils
import (
"bufio"
"github.com/up9inc/mizu/cli/mizu"
"fmt"
"log"
"os"
"strings"
@@ -11,7 +11,7 @@ import (
func AskForConfirmation(s string) bool {
reader := bufio.NewReader(os.Stdin)
mizu.Log.Infof(mizu.Magenta, s)
fmt.Printf(Magenta, s)
response, err := reader.ReadString('\n')
if err != nil {

View File

@@ -0,0 +1,36 @@
package uiUtils
import (
"bytes"
"encoding/json"
"gopkg.in/yaml.v3"
)
const (
empty = ""
tab = "\t"
)
func PrettyJson(data interface{}) (string, error) {
buffer := new(bytes.Buffer)
encoder := json.NewEncoder(buffer)
encoder.SetIndent(empty, tab)
err := encoder.Encode(data)
if err != nil {
return empty, err
}
return buffer.String(), nil
}
func PrettyYaml(data interface{}) (string, error) {
buffer := new(bytes.Buffer)
encoder := yaml.NewEncoder(buffer)
encoder.SetIndent(0)
err := encoder.Encode(data)
if err != nil {
return empty, err
}
return buffer.String(), nil
}