From 35dbd5fde2f7d5b8c4f41a99da2b11fce145fe56 Mon Sep 17 00:00:00 2001 From: RamiBerm <54766858+RamiBerm@users.noreply.github.com> Date: Sun, 31 Oct 2021 15:29:05 +0200 Subject: [PATCH] TRA-3860 create main configmap for agent and tappers (#410) * WIP * Update options.go and serializable_regexp.go * Update go.sum, go.sum, and 4 more files... * Update go.sum, go.sum, and 4 more files... * Update config.go and serializable_regexp.go * Update config.go, config.json, and test.go * Update tapRunner.go and provider.go * Update provider.go * Update tapRunner.go and provider.go * Update config.json and test.go * Update contract_validation.go, config.go, and 2 more files... * Update main.go * Update rulesHTTP.go * Update config.go, size_enforcer.go, and 5 more files... * Update config.go and config.go Co-authored-by: Rami Berman --- agent/main.go | 5 +++ agent/pkg/api/contract_validation.go | 2 +- agent/pkg/config/config.go | 57 ++++++++++++++++++++++++++++ agent/pkg/database/size_enforcer.go | 23 +---------- agent/pkg/rules/rulesHTTP.go | 2 +- cli/cmd/tapRunner.go | 24 +++++++----- cli/config/config.go | 25 ++++++++++++ cli/kubernetes/provider.go | 55 +++++++++++++++++---------- cli/mizu/consts.go | 2 +- shared/consts.go | 6 +-- shared/models.go | 5 +++ shared/serializable_regexp.go | 30 +++++++++++++++ 12 files changed, 179 insertions(+), 57 deletions(-) create mode 100644 agent/pkg/config/config.go create mode 100644 shared/serializable_regexp.go diff --git a/agent/main.go b/agent/main.go index 6ca1e76d2..324b9f277 100644 --- a/agent/main.go +++ b/agent/main.go @@ -6,6 +6,7 @@ import ( "fmt" "io/ioutil" "mizuserver/pkg/api" + "mizuserver/pkg/config" "mizuserver/pkg/controllers" "mizuserver/pkg/models" "mizuserver/pkg/routes" @@ -44,6 +45,9 @@ func main() { logLevel := determineLogLevel() logger.InitLoggerStderrOnly(logLevel) flag.Parse() + if err := config.LoadConfig(); err != nil { + logger.Log.Fatalf("Error loading config file %v", err) + } loadExtensions() if !*tapperMode && !*apiServerMode && !*standaloneMode && !*harsReaderMode { @@ -313,3 +317,4 @@ func determineLogLevel() (logLevel logging.Level) { } return } + diff --git a/agent/pkg/api/contract_validation.go b/agent/pkg/api/contract_validation.go index d57e40b38..5d575c938 100644 --- a/agent/pkg/api/contract_validation.go +++ b/agent/pkg/api/contract_validation.go @@ -24,7 +24,7 @@ const ( ) func loadOAS(ctx context.Context) (doc *openapi3.T, contractContent string, router routers.Router, err error) { - path := fmt.Sprintf("%s/%s", shared.RulePolicyPath, shared.ContractFileName) + path := fmt.Sprintf("%s%s", shared.ConfigDirPath, shared.ContractFileName) bytes, err := ioutil.ReadFile(path) if err != nil { logger.Log.Error(err.Error()) diff --git a/agent/pkg/config/config.go b/agent/pkg/config/config.go new file mode 100644 index 000000000..f5cdb320a --- /dev/null +++ b/agent/pkg/config/config.go @@ -0,0 +1,57 @@ +package config + +import ( + "encoding/json" + "fmt" + "github.com/up9inc/mizu/shared" + "io/ioutil" + "os" +) + +// these values are used when the config.json file is not present +const ( + defaultMaxDatabaseSizeBytes int64 = 200 * 1000 * 1000 + defaultRegexTarget string = ".*" +) + +var Config *shared.MizuAgentConfig + +func LoadConfig() error { + if Config != nil { + return nil + } + filePath := fmt.Sprintf("%s%s", shared.ConfigDirPath, shared.ConfigFileName) + + content, err := ioutil.ReadFile(filePath) + if err != nil { + if os.IsNotExist(err) { + return applyDefaultConfig() + } + return err + } + + if err = json.Unmarshal(content, &Config); err != nil { + return err + } + return nil +} + +func applyDefaultConfig() error { + defaultConfig, err := getDefaultConfig() + if err != nil { + return err + } + Config = defaultConfig + return nil +} + +func getDefaultConfig() (*shared.MizuAgentConfig, error) { + regex, err := shared.CompileRegexToSerializableRegexp(defaultRegexTarget) + if err != nil { + return nil, err + } + return &shared.MizuAgentConfig{ + TapTargetRegex: *regex, + MaxDBSizeBytes: defaultMaxDatabaseSizeBytes, + }, nil +} diff --git a/agent/pkg/database/size_enforcer.go b/agent/pkg/database/size_enforcer.go index 4ed87dddd..7e06da7fe 100644 --- a/agent/pkg/database/size_enforcer.go +++ b/agent/pkg/database/size_enforcer.go @@ -1,12 +1,11 @@ package database import ( + "mizuserver/pkg/config" "os" - "strconv" "time" "github.com/fsnotify/fsnotify" - "github.com/up9inc/mizu/shared" "github.com/up9inc/mizu/shared/debounce" "github.com/up9inc/mizu/shared/logger" "github.com/up9inc/mizu/shared/units" @@ -14,7 +13,6 @@ import ( ) const percentageOfMaxSizeBytesToPrune = 15 -const defaultMaxDatabaseSizeBytes int64 = 200 * 1000 * 1000 func StartEnforcingDatabaseSize() { watcher, err := fsnotify.NewWatcher() @@ -23,14 +21,8 @@ func StartEnforcingDatabaseSize() { return } - maxEntriesDBByteSize, err := getMaxEntriesDBByteSize() - if err != nil { - logger.Log.Fatalf("Error parsing max db size: %v\n", err) - return - } - checkFileSizeDebouncer := debounce.NewDebouncer(5*time.Second, func() { - checkFileSize(maxEntriesDBByteSize) + checkFileSize(config.Config.MaxDBSizeBytes) }) go func() { @@ -58,17 +50,6 @@ func StartEnforcingDatabaseSize() { } } -func getMaxEntriesDBByteSize() (int64, error) { - maxEntriesDBByteSize := defaultMaxDatabaseSizeBytes - var err error - - maxEntriesDBSizeByteSEnvVarValue := os.Getenv(shared.MaxEntriesDBSizeBytesEnvVar) - if maxEntriesDBSizeByteSEnvVarValue != "" { - maxEntriesDBByteSize, err = strconv.ParseInt(maxEntriesDBSizeByteSEnvVarValue, 10, 64) - } - return maxEntriesDBByteSize, err -} - func checkFileSize(maxSizeBytes int64) { fileStat, err := os.Stat(DBPath) if err != nil { diff --git a/agent/pkg/rules/rulesHTTP.go b/agent/pkg/rules/rulesHTTP.go index 4bb92fe6c..61550fbcf 100644 --- a/agent/pkg/rules/rulesHTTP.go +++ b/agent/pkg/rules/rulesHTTP.go @@ -45,7 +45,7 @@ func ValidateService(serviceFromRule string, service string) bool { } func MatchRequestPolicy(harEntry har.Entry, service string) (resultPolicyToSend []RulesMatched, isEnabled bool) { - enforcePolicy, err := shared.DecodeEnforcePolicy(fmt.Sprintf("%s/%s", shared.RulePolicyPath, shared.RulePolicyFileName)) + enforcePolicy, err := shared.DecodeEnforcePolicy(fmt.Sprintf("%s%s", shared.ConfigDirPath, shared.ValidationRulesFileName)) if err == nil && len(enforcePolicy.Rules) > 0 { isEnabled = true } diff --git a/cli/cmd/tapRunner.go b/cli/cmd/tapRunner.go index 006ea1a11..f6ad1268c 100644 --- a/cli/cmd/tapRunner.go +++ b/cli/cmd/tapRunner.go @@ -54,9 +54,9 @@ func RunMizuTap() { return } - var mizuValidationRules string + var serializedValidationRules string if config.Config.Tap.EnforcePolicyFile != "" { - mizuValidationRules, err = readValidationRules(config.Config.Tap.EnforcePolicyFile) + serializedValidationRules, err = readValidationRules(config.Config.Tap.EnforcePolicyFile) if err != nil { logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error reading policy file: %v", errormessage.FormatError(err))) return @@ -64,14 +64,14 @@ func RunMizuTap() { } // Read and validate the OAS file - var contract string + var serializedContract string if config.Config.Tap.ContractFile != "" { bytes, err := ioutil.ReadFile(config.Config.Tap.ContractFile) if err != nil { logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error reading contract file: %v", errormessage.FormatError(err))) return } - contract = string(bytes) + serializedContract = string(bytes) ctx := context.Background() loader := &openapi3.Loader{Context: ctx} @@ -87,6 +87,12 @@ func RunMizuTap() { } } + serializedMizuConfig, err := config.GetSerializedMizuConfig() + if err != nil { + logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error composing mizu config: %v", errormessage.FormatError(err))) + return + } + kubernetesProvider, err := kubernetes.NewProvider(config.Config.KubeConfigPath()) if err != nil { logger.Log.Error(err) @@ -132,7 +138,7 @@ func RunMizuTap() { return } - if err := createMizuResources(ctx, kubernetesProvider, mizuValidationRules, contract); err != nil { + if err := createMizuResources(ctx, kubernetesProvider, serializedValidationRules, serializedContract, serializedMizuConfig); err != nil { logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error creating resources: %v", errormessage.FormatError(err))) var statusError *k8serrors.StatusError @@ -162,7 +168,7 @@ func readValidationRules(file string) (string, error) { return string(newContent), nil } -func createMizuResources(ctx context.Context, kubernetesProvider *kubernetes.Provider, mizuValidationRules string, contract string) error { +func createMizuResources(ctx context.Context, kubernetesProvider *kubernetes.Provider, serializedValidationRules string, serializedContract string, serializedMizuConfig string) error { if !config.Config.IsNsRestrictedMode() { if err := createMizuNamespace(ctx, kubernetesProvider); err != nil { return err @@ -173,15 +179,15 @@ func createMizuResources(ctx context.Context, kubernetesProvider *kubernetes.Pro return err } - if err := createMizuConfigmap(ctx, kubernetesProvider, mizuValidationRules, contract); err != nil { + if err := createMizuConfigmap(ctx, kubernetesProvider, serializedValidationRules, serializedContract, serializedMizuConfig); err != nil { logger.Log.Warningf(uiUtils.Warning, fmt.Sprintf("Failed to create resources required for policy validation. Mizu will not validate policy rules. error: %v\n", errormessage.FormatError(err))) } return nil } -func createMizuConfigmap(ctx context.Context, kubernetesProvider *kubernetes.Provider, data string, contract string) error { - err := kubernetesProvider.CreateConfigMap(ctx, config.Config.MizuResourcesNamespace, mizu.ConfigMapName, data, contract) +func createMizuConfigmap(ctx context.Context, kubernetesProvider *kubernetes.Provider, serializedValidationRules string, serializedContract string, serializedMizuConfig string) error { + err := kubernetesProvider.CreateConfigMap(ctx, config.Config.MizuResourcesNamespace, mizu.ConfigMapName, serializedValidationRules, serializedContract, serializedMizuConfig) return err } diff --git a/cli/config/config.go b/cli/config/config.go index 64632fea9..ae8d172f2 100644 --- a/cli/config/config.go +++ b/cli/config/config.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "io/ioutil" + "k8s.io/apimachinery/pkg/util/json" "os" "reflect" "strconv" @@ -363,3 +364,27 @@ func setZeroForReadonlyFields(currentElem reflect.Value) { } } } + +func GetSerializedMizuConfig() (string, error) { + mizuConfig, err := getMizuConfig() + if err != nil { + return "", err + } + serializedConfig, err := json.Marshal(mizuConfig) + if err != nil { + return "", err + } + return string(serializedConfig), nil +} + +func getMizuConfig() (*shared.MizuAgentConfig, error) { + serializableRegex, err := shared.CompileRegexToSerializableRegexp(Config.Tap.PodRegexStr) + if err != nil { + return nil, err + } + config := shared.MizuAgentConfig{ + TapTargetRegex: *serializableRegex, + MaxDBSizeBytes: Config.Tap.MaxEntriesDBSizeBytes(), + } + return &config, nil +} diff --git a/cli/kubernetes/provider.go b/cli/kubernetes/provider.go index 3b1bc9773..c2514a581 100644 --- a/cli/kubernetes/provider.go +++ b/cli/kubernetes/provider.go @@ -7,15 +7,13 @@ import ( "encoding/json" "errors" "fmt" + "github.com/up9inc/mizu/cli/config/configStructs" + "github.com/up9inc/mizu/shared/logger" "github.com/up9inc/mizu/shared/semver" "k8s.io/apimachinery/pkg/version" "net/url" "path/filepath" "regexp" - "strconv" - - "github.com/up9inc/mizu/cli/config/configStructs" - "github.com/up9inc/mizu/shared/logger" "io" @@ -178,10 +176,8 @@ func (provider *Provider) CreateMizuApiServerPod(ctx context.Context, opts *ApiS } } - configMapVolumeName := &core.ConfigMapVolumeSource{} - configMapVolumeName.Name = mizu.ConfigMapName - configMapOptional := true - configMapVolumeName.Optional = &configMapOptional + configMapVolume := &core.ConfigMapVolumeSource{} + configMapVolume.Name = mizu.ConfigMapName cpuLimit, err := resource.ParseQuantity(opts.Resources.CpuLimit) if err != nil { @@ -227,7 +223,7 @@ func (provider *Provider) CreateMizuApiServerPod(ctx context.Context, opts *ApiS VolumeMounts: []core.VolumeMount{ { Name: mizu.ConfigMapName, - MountPath: shared.RulePolicyPath, + MountPath: shared.ConfigDirPath, }, }, Command: command, @@ -236,10 +232,6 @@ func (provider *Provider) CreateMizuApiServerPod(ctx context.Context, opts *ApiS Name: shared.SyncEntriesConfigEnvVar, Value: string(marshaledSyncEntriesConfig), }, - { - Name: shared.MaxEntriesDBSizeBytesEnvVar, - Value: strconv.FormatInt(opts.MaxEntriesDBSizeBytes, 10), - }, { Name: shared.DebugModeEnvVar, Value: debugMode, @@ -280,7 +272,7 @@ func (provider *Provider) CreateMizuApiServerPod(ctx context.Context, opts *ApiS { Name: mizu.ConfigMapName, VolumeSource: core.VolumeSource{ - ConfigMap: configMapVolumeName, + ConfigMap: configMapVolume, }, }, }, @@ -496,14 +488,16 @@ func (provider *Provider) handleRemovalError(err error) error { return err } -func (provider *Provider) CreateConfigMap(ctx context.Context, namespace string, configMapName string, data string, contract string) error { - if data == "" && contract == "" { - return nil - } - +func (provider *Provider) CreateConfigMap(ctx context.Context, namespace string, configMapName string, serializedValidationRules string, serializedContract string, serializedMizuConfig string) error { configMapData := make(map[string]string, 0) - configMapData[shared.RulePolicyFileName] = data - configMapData[shared.ContractFileName] = contract + if serializedValidationRules != "" { + configMapData[shared.ValidationRulesFileName] = serializedValidationRules + } + if serializedContract != "" { + configMapData[shared.ContractFileName] = serializedContract + } + configMapData[shared.ConfigFileName] = serializedMizuConfig + configMap := &core.ConfigMap{ TypeMeta: metav1.TypeMeta{ Kind: "ConfigMap", @@ -622,6 +616,24 @@ func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespac noScheduleToleration.WithOperator(core.TolerationOpExists) noScheduleToleration.WithEffect(core.TaintEffectNoSchedule) + volumeName := mizu.ConfigMapName + configMapVolume := applyconfcore.VolumeApplyConfiguration{ + Name: &volumeName, + VolumeSourceApplyConfiguration: applyconfcore.VolumeSourceApplyConfiguration{ + ConfigMap: &applyconfcore.ConfigMapVolumeSourceApplyConfiguration{ + LocalObjectReferenceApplyConfiguration: applyconfcore.LocalObjectReferenceApplyConfiguration{ + Name: &volumeName, + }, + }, + }, + } + mountPath := shared.ConfigDirPath + configMapVolumeMount := applyconfcore.VolumeMountApplyConfiguration{ + Name: &volumeName, + MountPath: &mountPath, + } + agentContainer.WithVolumeMounts(&configMapVolumeMount) + podSpec := applyconfcore.PodSpec() podSpec.WithHostNetwork(true) podSpec.WithDNSPolicy(core.DNSClusterFirstWithHostNet) @@ -632,6 +644,7 @@ func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespac podSpec.WithContainers(agentContainer) podSpec.WithAffinity(affinity) podSpec.WithTolerations(noExecuteToleration, noScheduleToleration) + podSpec.WithVolumes(&configMapVolume) podTemplate := applyconfcore.PodTemplateSpec() podTemplate.WithLabels(map[string]string{"app": tapperPodName}) diff --git a/cli/mizu/consts.go b/cli/mizu/consts.go index a25e6733b..327254341 100644 --- a/cli/mizu/consts.go +++ b/cli/mizu/consts.go @@ -24,7 +24,7 @@ const ( ServiceAccountName = MizuResourcesPrefix + "service-account" TapperDaemonSetName = MizuResourcesPrefix + "tapper-daemon-set" TapperPodName = MizuResourcesPrefix + "tapper" - ConfigMapName = MizuResourcesPrefix + "policy" + ConfigMapName = MizuResourcesPrefix + "config" MinKubernetesServerVersion = "1.16.0" ) diff --git a/shared/consts.go b/shared/consts.go index 5f4467c1d..84efe1219 100644 --- a/shared/consts.go +++ b/shared/consts.go @@ -6,10 +6,10 @@ const ( HostModeEnvVar = "HOST_MODE" NodeNameEnvVar = "NODE_NAME" TappedAddressesPerNodeDictEnvVar = "TAPPED_ADDRESSES_PER_HOST" - MaxEntriesDBSizeBytesEnvVar = "MAX_ENTRIES_DB_BYTES" - RulePolicyPath = "/app/enforce-policy/" - RulePolicyFileName = "enforce-policy.yaml" + ConfigDirPath = "/app/config/" + ValidationRulesFileName = "validation-rules.yaml" ContractFileName = "contract-oas.yaml" + ConfigFileName = "mizu-config.json" GoGCEnvVar = "GOGC" DefaultApiServerPort = 8899 DebugModeEnvVar = "MIZU_DEBUG" diff --git a/shared/models.go b/shared/models.go index 2577dd4cc..44209b747 100644 --- a/shared/models.go +++ b/shared/models.go @@ -18,6 +18,11 @@ const ( WebsocketMessageTypeOutboundLink WebSocketMessageType = "outboundLink" ) +type MizuAgentConfig struct { + TapTargetRegex SerializableRegexp `yaml:"tapTargetRegex"` + MaxDBSizeBytes int64 `yaml:"maxDBSizeBytes"` +} + type WebSocketMessageMetadata struct { MessageType WebSocketMessageType `json:"messageType,omitempty"` } diff --git a/shared/serializable_regexp.go b/shared/serializable_regexp.go new file mode 100644 index 000000000..e311fdeb5 --- /dev/null +++ b/shared/serializable_regexp.go @@ -0,0 +1,30 @@ +package shared + +import "regexp" + +type SerializableRegexp struct { + regexp.Regexp +} + +func CompileRegexToSerializableRegexp(expr string) (*SerializableRegexp, error) { + re, err := regexp.Compile(expr) + if err != nil { + return nil, err + } + return &SerializableRegexp{*re}, nil +} + +// UnmarshalText is by json.Unmarshal. +func (r *SerializableRegexp) UnmarshalText(text []byte) error { + rr, err := CompileRegexToSerializableRegexp(string(text)) + if err != nil { + return err + } + *r = *rr + return nil +} + +// MarshalText is used by json.Marshal. +func (r *SerializableRegexp) MarshalText() ([]byte, error) { + return []byte(r.String()), nil +}