TRA-3860 create main configmap for agent and tappers (#410)

* WIP

* Update options.go and serializable_regexp.go

* Update go.sum, go.sum, and 4 more files...

* Update go.sum, go.sum, and 4 more files...

* Update config.go and serializable_regexp.go

* Update config.go, config.json, and test.go

* Update tapRunner.go and provider.go

* Update provider.go

* Update tapRunner.go and provider.go

* Update config.json and test.go

* Update contract_validation.go, config.go, and 2 more files...

* Update main.go

* Update rulesHTTP.go

* Update config.go, size_enforcer.go, and 5 more files...

* Update config.go and config.go

Co-authored-by: Rami Berman <rami.berman@up9.com>

agent/main.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"mizuserver/pkg/api"
+	"mizuserver/pkg/config"
 	"mizuserver/pkg/controllers"
 	"mizuserver/pkg/models"
 	"mizuserver/pkg/routes"
@@ -44,6 +45,9 @@ func main() {
 	logLevel := determineLogLevel()
 	logger.InitLoggerStderrOnly(logLevel)
 	flag.Parse()
+	if err := config.LoadConfig(); err != nil {
+		logger.Log.Fatalf("Error loading config file %v", err)
+	}
 	loadExtensions()
 
 	if !*tapperMode && !*apiServerMode && !*standaloneMode && !*harsReaderMode {
@@ -313,3 +317,4 @@ func determineLogLevel() (logLevel logging.Level) {
 	}
 	return
 }
+

agent/pkg/api/contract_validation.go

@@ -24,7 +24,7 @@ const (
 )
 
 func loadOAS(ctx context.Context) (doc *openapi3.T, contractContent string, router routers.Router, err error) {
-	path := fmt.Sprintf("%s/%s", shared.RulePolicyPath, shared.ContractFileName)
+	path := fmt.Sprintf("%s%s", shared.ConfigDirPath, shared.ContractFileName)
 	bytes, err := ioutil.ReadFile(path)
 	if err != nil {
 		logger.Log.Error(err.Error())

agent/pkg/config/config.go

@@ -0,0 +1,57 @@
+package config
+
+import (
+	"encoding/json"
+	"fmt"
+	"github.com/up9inc/mizu/shared"
+	"io/ioutil"
+	"os"
+)
+
+// these values are used when the config.json file is not present
+const (
+	defaultMaxDatabaseSizeBytes int64  = 200 * 1000 * 1000
+	defaultRegexTarget          string = ".*"
+)
+
+var Config *shared.MizuAgentConfig
+
+func LoadConfig() error {
+	if Config != nil {
+		return nil
+	}
+	filePath := fmt.Sprintf("%s%s", shared.ConfigDirPath, shared.ConfigFileName)
+
+	content, err := ioutil.ReadFile(filePath)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return applyDefaultConfig()
+		}
+		return err
+	}
+
+	if err = json.Unmarshal(content, &Config); err != nil {
+		return err
+	}
+	return nil
+}
+
+func applyDefaultConfig() error {
+	defaultConfig, err := getDefaultConfig()
+	if err != nil {
+		return err
+	}
+	Config = defaultConfig
+	return nil
+}
+
+func getDefaultConfig() (*shared.MizuAgentConfig, error) {
+	regex, err := shared.CompileRegexToSerializableRegexp(defaultRegexTarget)
+	if err != nil {
+		return nil, err
+	}
+	return &shared.MizuAgentConfig{
+		TapTargetRegex: *regex,
+		MaxDBSizeBytes: defaultMaxDatabaseSizeBytes,
+	}, nil
+}

agent/pkg/database/size_enforcer.go

@@ -1,12 +1,11 @@
 package database
 
 import (
+	"mizuserver/pkg/config"
 	"os"
-	"strconv"
 	"time"
 
 	"github.com/fsnotify/fsnotify"
-	"github.com/up9inc/mizu/shared"
 	"github.com/up9inc/mizu/shared/debounce"
 	"github.com/up9inc/mizu/shared/logger"
 	"github.com/up9inc/mizu/shared/units"
@@ -14,7 +13,6 @@ import (
 )
 
 const percentageOfMaxSizeBytesToPrune = 15
-const defaultMaxDatabaseSizeBytes int64 = 200 * 1000 * 1000
 
 func StartEnforcingDatabaseSize() {
 	watcher, err := fsnotify.NewWatcher()
@@ -23,14 +21,8 @@ func StartEnforcingDatabaseSize() {
 		return
 	}
 
-	maxEntriesDBByteSize, err := getMaxEntriesDBByteSize()
-	if err != nil {
-		logger.Log.Fatalf("Error parsing max db size: %v\n", err)
-		return
-	}
-
 	checkFileSizeDebouncer := debounce.NewDebouncer(5*time.Second, func() {
-		checkFileSize(maxEntriesDBByteSize)
+		checkFileSize(config.Config.MaxDBSizeBytes)
 	})
 
 	go func() {
@@ -58,17 +50,6 @@ func StartEnforcingDatabaseSize() {
 	}
 }
 
-func getMaxEntriesDBByteSize() (int64, error) {
-	maxEntriesDBByteSize := defaultMaxDatabaseSizeBytes
-	var err error
-
-	maxEntriesDBSizeByteSEnvVarValue := os.Getenv(shared.MaxEntriesDBSizeBytesEnvVar)
-	if maxEntriesDBSizeByteSEnvVarValue != "" {
-		maxEntriesDBByteSize, err = strconv.ParseInt(maxEntriesDBSizeByteSEnvVarValue, 10, 64)
-	}
-	return maxEntriesDBByteSize, err
-}
-
 func checkFileSize(maxSizeBytes int64) {
 	fileStat, err := os.Stat(DBPath)
 	if err != nil {

agent/pkg/rules/rulesHTTP.go

@@ -45,7 +45,7 @@ func ValidateService(serviceFromRule string, service string) bool {
 }
 
 func MatchRequestPolicy(harEntry har.Entry, service string) (resultPolicyToSend []RulesMatched, isEnabled bool) {
-	enforcePolicy, err := shared.DecodeEnforcePolicy(fmt.Sprintf("%s/%s", shared.RulePolicyPath, shared.RulePolicyFileName))
+	enforcePolicy, err := shared.DecodeEnforcePolicy(fmt.Sprintf("%s%s", shared.ConfigDirPath, shared.ValidationRulesFileName))
 	if err == nil && len(enforcePolicy.Rules) > 0 {
 		isEnabled = true
 	}

cli/cmd/tapRunner.go

@@ -54,9 +54,9 @@ func RunMizuTap() {
 		return
 	}
 
-	var mizuValidationRules string
+	var serializedValidationRules string
 	if config.Config.Tap.EnforcePolicyFile != "" {
-		mizuValidationRules, err = readValidationRules(config.Config.Tap.EnforcePolicyFile)
+		serializedValidationRules, err = readValidationRules(config.Config.Tap.EnforcePolicyFile)
 		if err != nil {
 			logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error reading policy file: %v", errormessage.FormatError(err)))
 			return
@@ -64,14 +64,14 @@ func RunMizuTap() {
 	}
 
 	// Read and validate the OAS file
-	var contract string
+	var serializedContract string
 	if config.Config.Tap.ContractFile != "" {
 		bytes, err := ioutil.ReadFile(config.Config.Tap.ContractFile)
 		if err != nil {
 			logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error reading contract file: %v", errormessage.FormatError(err)))
 			return
 		}
-		contract = string(bytes)
+		serializedContract = string(bytes)
 
 		ctx := context.Background()
 		loader := &openapi3.Loader{Context: ctx}
@@ -87,6 +87,12 @@ func RunMizuTap() {
 		}
 	}
 
+	serializedMizuConfig, err := config.GetSerializedMizuConfig()
+	if err != nil {
+		logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error composing mizu config: %v", errormessage.FormatError(err)))
+		return
+	}
+
 	kubernetesProvider, err := kubernetes.NewProvider(config.Config.KubeConfigPath())
 	if err != nil {
 		logger.Log.Error(err)
@@ -132,7 +138,7 @@ func RunMizuTap() {
 		return
 	}
 
-	if err := createMizuResources(ctx, kubernetesProvider, mizuValidationRules, contract); err != nil {
+	if err := createMizuResources(ctx, kubernetesProvider, serializedValidationRules, serializedContract, serializedMizuConfig); err != nil {
 		logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error creating resources: %v", errormessage.FormatError(err)))
 
 		var statusError *k8serrors.StatusError
@@ -162,7 +168,7 @@ func readValidationRules(file string) (string, error) {
 	return string(newContent), nil
 }
 
-func createMizuResources(ctx context.Context, kubernetesProvider *kubernetes.Provider, mizuValidationRules string, contract string) error {
+func createMizuResources(ctx context.Context, kubernetesProvider *kubernetes.Provider, serializedValidationRules string, serializedContract string, serializedMizuConfig string) error {
 	if !config.Config.IsNsRestrictedMode() {
 		if err := createMizuNamespace(ctx, kubernetesProvider); err != nil {
 			return err
@@ -173,15 +179,15 @@ func createMizuResources(ctx context.Context, kubernetesProvider *kubernetes.Pro
 		return err
 	}
 
-	if err := createMizuConfigmap(ctx, kubernetesProvider, mizuValidationRules, contract); err != nil {
+	if err := createMizuConfigmap(ctx, kubernetesProvider, serializedValidationRules, serializedContract, serializedMizuConfig); err != nil {
 		logger.Log.Warningf(uiUtils.Warning, fmt.Sprintf("Failed to create resources required for policy validation. Mizu will not validate policy rules. error: %v\n", errormessage.FormatError(err)))
 	}
 
 	return nil
 }
 
-func createMizuConfigmap(ctx context.Context, kubernetesProvider *kubernetes.Provider, data string, contract string) error {
+func createMizuConfigmap(ctx context.Context, kubernetesProvider *kubernetes.Provider, serializedValidationRules string, serializedContract string, serializedMizuConfig string) error {
-	err := kubernetesProvider.CreateConfigMap(ctx, config.Config.MizuResourcesNamespace, mizu.ConfigMapName, data, contract)
+	err := kubernetesProvider.CreateConfigMap(ctx, config.Config.MizuResourcesNamespace, mizu.ConfigMapName, serializedValidationRules, serializedContract, serializedMizuConfig)
 	return err
 }
 

cli/config/config.go

@@ -4,6 +4,7 @@ import (
 	"errors"
 	"fmt"
 	"io/ioutil"
+	"k8s.io/apimachinery/pkg/util/json"
 	"os"
 	"reflect"
 	"strconv"
@@ -363,3 +364,27 @@ func setZeroForReadonlyFields(currentElem reflect.Value) {
 		}
 	}
 }
+
+func GetSerializedMizuConfig() (string, error) {
+	mizuConfig, err := getMizuConfig()
+	if err != nil {
+		return "", err
+	}
+	serializedConfig, err := json.Marshal(mizuConfig)
+	if err != nil {
+		return "", err
+	}
+	return string(serializedConfig), nil
+}
+
+func getMizuConfig() (*shared.MizuAgentConfig, error) {
+	serializableRegex, err := shared.CompileRegexToSerializableRegexp(Config.Tap.PodRegexStr)
+	if err != nil {
+		return nil, err
+	}
+	config := shared.MizuAgentConfig{
+		TapTargetRegex: *serializableRegex,
+		MaxDBSizeBytes: Config.Tap.MaxEntriesDBSizeBytes(),
+	}
+	return &config, nil
+}

cli/kubernetes/provider.go

@@ -7,15 +7,13 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"github.com/up9inc/mizu/cli/config/configStructs"
+	"github.com/up9inc/mizu/shared/logger"
 	"github.com/up9inc/mizu/shared/semver"
 	"k8s.io/apimachinery/pkg/version"
 	"net/url"
 	"path/filepath"
 	"regexp"
-	"strconv"
-
-	"github.com/up9inc/mizu/cli/config/configStructs"
-	"github.com/up9inc/mizu/shared/logger"
 
 	"io"
 
@@ -178,10 +176,8 @@ func (provider *Provider) CreateMizuApiServerPod(ctx context.Context, opts *ApiS
 		}
 	}
 
-	configMapVolumeName := &core.ConfigMapVolumeSource{}
+	configMapVolume := &core.ConfigMapVolumeSource{}
-	configMapVolumeName.Name = mizu.ConfigMapName
+	configMapVolume.Name = mizu.ConfigMapName
-	configMapOptional := true
-	configMapVolumeName.Optional = &configMapOptional
 
 	cpuLimit, err := resource.ParseQuantity(opts.Resources.CpuLimit)
 	if err != nil {
@@ -227,7 +223,7 @@ func (provider *Provider) CreateMizuApiServerPod(ctx context.Context, opts *ApiS
 					VolumeMounts: []core.VolumeMount{
 						{
 							Name:      mizu.ConfigMapName,
-							MountPath: shared.RulePolicyPath,
+							MountPath: shared.ConfigDirPath,
 						},
 					},
 					Command: command,
@@ -236,10 +232,6 @@ func (provider *Provider) CreateMizuApiServerPod(ctx context.Context, opts *ApiS
 							Name:  shared.SyncEntriesConfigEnvVar,
 							Value: string(marshaledSyncEntriesConfig),
 						},
-						{
-							Name:  shared.MaxEntriesDBSizeBytesEnvVar,
-							Value: strconv.FormatInt(opts.MaxEntriesDBSizeBytes, 10),
-						},
 						{
 							Name:  shared.DebugModeEnvVar,
 							Value: debugMode,
@@ -280,7 +272,7 @@ func (provider *Provider) CreateMizuApiServerPod(ctx context.Context, opts *ApiS
 				{
 					Name: mizu.ConfigMapName,
 					VolumeSource: core.VolumeSource{
-						ConfigMap: configMapVolumeName,
+						ConfigMap: configMapVolume,
 					},
 				},
 			},
@@ -496,14 +488,16 @@ func (provider *Provider) handleRemovalError(err error) error {
 	return err
 }
 
-func (provider *Provider) CreateConfigMap(ctx context.Context, namespace string, configMapName string, data string, contract string) error {
+func (provider *Provider) CreateConfigMap(ctx context.Context, namespace string, configMapName string, serializedValidationRules string, serializedContract string, serializedMizuConfig string) error {
-	if data == "" && contract == "" {
-		return nil
-	}
-
 	configMapData := make(map[string]string, 0)
-	configMapData[shared.RulePolicyFileName] = data
+	if serializedValidationRules != "" {
-	configMapData[shared.ContractFileName] = contract
+		configMapData[shared.ValidationRulesFileName] = serializedValidationRules
+	}
+	if serializedContract != "" {
+		configMapData[shared.ContractFileName] = serializedContract
+	}
+	configMapData[shared.ConfigFileName] = serializedMizuConfig
+
 	configMap := &core.ConfigMap{
 		TypeMeta: metav1.TypeMeta{
 			Kind:       "ConfigMap",
@@ -622,6 +616,24 @@ func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespac
 	noScheduleToleration.WithOperator(core.TolerationOpExists)
 	noScheduleToleration.WithEffect(core.TaintEffectNoSchedule)
 
+	volumeName := mizu.ConfigMapName
+	configMapVolume := applyconfcore.VolumeApplyConfiguration{
+		Name:                           &volumeName,
+		VolumeSourceApplyConfiguration: applyconfcore.VolumeSourceApplyConfiguration{
+			ConfigMap: &applyconfcore.ConfigMapVolumeSourceApplyConfiguration{
+				LocalObjectReferenceApplyConfiguration: applyconfcore.LocalObjectReferenceApplyConfiguration{
+					Name: &volumeName,
+				},
+			},
+		},
+	}
+	mountPath := shared.ConfigDirPath
+	configMapVolumeMount := applyconfcore.VolumeMountApplyConfiguration{
+		Name:             &volumeName,
+		MountPath:        &mountPath,
+	}
+	agentContainer.WithVolumeMounts(&configMapVolumeMount)
+
 	podSpec := applyconfcore.PodSpec()
 	podSpec.WithHostNetwork(true)
 	podSpec.WithDNSPolicy(core.DNSClusterFirstWithHostNet)
@@ -632,6 +644,7 @@ func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespac
 	podSpec.WithContainers(agentContainer)
 	podSpec.WithAffinity(affinity)
 	podSpec.WithTolerations(noExecuteToleration, noScheduleToleration)
+	podSpec.WithVolumes(&configMapVolume)
 
 	podTemplate := applyconfcore.PodTemplateSpec()
 	podTemplate.WithLabels(map[string]string{"app": tapperPodName})

cli/mizu/consts.go

@@ -24,7 +24,7 @@ const (
 	ServiceAccountName         = MizuResourcesPrefix + "service-account"
 	TapperDaemonSetName        = MizuResourcesPrefix + "tapper-daemon-set"
 	TapperPodName              = MizuResourcesPrefix + "tapper"
-	ConfigMapName              = MizuResourcesPrefix + "policy"
+	ConfigMapName              = MizuResourcesPrefix + "config"
 	MinKubernetesServerVersion = "1.16.0"
 )
 

shared/consts.go

@@ -6,10 +6,10 @@ const (
 	HostModeEnvVar                   = "HOST_MODE"
 	NodeNameEnvVar                   = "NODE_NAME"
 	TappedAddressesPerNodeDictEnvVar = "TAPPED_ADDRESSES_PER_HOST"
-	MaxEntriesDBSizeBytesEnvVar      = "MAX_ENTRIES_DB_BYTES"
+	ConfigDirPath                    = "/app/config/"
-	RulePolicyPath                   = "/app/enforce-policy/"
+	ValidationRulesFileName          = "validation-rules.yaml"
-	RulePolicyFileName               = "enforce-policy.yaml"
 	ContractFileName                 = "contract-oas.yaml"
+	ConfigFileName                   = "mizu-config.json"
 	GoGCEnvVar                       = "GOGC"
 	DefaultApiServerPort             = 8899
 	DebugModeEnvVar                  = "MIZU_DEBUG"

shared/models.go

@@ -18,6 +18,11 @@ const (
 	WebsocketMessageTypeOutboundLink  WebSocketMessageType = "outboundLink"
 )
 
+type MizuAgentConfig struct {
+	TapTargetRegex SerializableRegexp `yaml:"tapTargetRegex"`
+	MaxDBSizeBytes int64              `yaml:"maxDBSizeBytes"`
+}
+
 type WebSocketMessageMetadata struct {
 	MessageType WebSocketMessageType `json:"messageType,omitempty"`
 }

shared/serializable_regexp.go

@@ -0,0 +1,30 @@
+package shared
+
+import "regexp"
+
+type SerializableRegexp struct {
+	regexp.Regexp
+}
+
+func CompileRegexToSerializableRegexp(expr string) (*SerializableRegexp, error) {
+	re, err := regexp.Compile(expr)
+	if err != nil {
+		return nil, err
+	}
+	return &SerializableRegexp{*re}, nil
+}
+
+// UnmarshalText is by json.Unmarshal.
+func (r *SerializableRegexp) UnmarshalText(text []byte) error {
+	rr, err := CompileRegexToSerializableRegexp(string(text))
+	if err != nil {
+		return err
+	}
+	*r = *rr
+	return nil
+}
+
+// MarshalText is used by json.Marshal.
+func (r *SerializableRegexp) MarshalText() ([]byte, error) {
+	return []byte(r.String()), nil
+}