From 26b3bacb653fca606bd5b9db0e5f7391c0adaa9f Mon Sep 17 00:00:00 2001 From: wojtekt Date: Fri, 6 Jul 2018 12:30:36 +0200 Subject: [PATCH] Make proxier params configurable in kubemark --- cluster/kubemark/gce/config-default.sh | 10 ++++---- .../kubemark/pre-existing/config-default.sh | 1 - cmd/kubemark/hollow-node.go | 24 ++++++++++++------- pkg/kubemark/hollow_proxy.go | 6 +++-- .../resources/hollow-node_template.yaml | 4 ++-- test/kubemark/start-kubemark.sh | 5 ++-- 6 files changed, 29 insertions(+), 21 deletions(-) diff --git a/cluster/kubemark/gce/config-default.sh b/cluster/kubemark/gce/config-default.sh index c58350e70bb..659a57653d8 100644 --- a/cluster/kubemark/gce/config-default.sh +++ b/cluster/kubemark/gce/config-default.sh @@ -73,11 +73,12 @@ STORAGE_MEDIA_TYPE=${STORAGE_MEDIA_TYPE:-} # Default Log level for all components in test clusters and variables to override it in specific components. TEST_CLUSTER_LOG_LEVEL="${TEST_CLUSTER_LOG_LEVEL:---v=4}" -KUBELET_TEST_LOG_LEVEL="${KUBELET_TEST_LOG_LEVEL:-$TEST_CLUSTER_LOG_LEVEL}" API_SERVER_TEST_LOG_LEVEL="${API_SERVER_TEST_LOG_LEVEL:-$TEST_CLUSTER_LOG_LEVEL}" CONTROLLER_MANAGER_TEST_LOG_LEVEL="${CONTROLLER_MANAGER_TEST_LOG_LEVEL:-$TEST_CLUSTER_LOG_LEVEL}" SCHEDULER_TEST_LOG_LEVEL="${SCHEDULER_TEST_LOG_LEVEL:-$TEST_CLUSTER_LOG_LEVEL}" -KUBEPROXY_TEST_LOG_LEVEL="${KUBEPROXY_TEST_LOG_LEVEL:-$TEST_CLUSTER_LOG_LEVEL}" + +HOLLOW_KUBELET_TEST_LOG_LEVEL="${HOLLOW_KUBELET_TEST_LOG_LEVEL:-$TEST_CLUSTER_LOG_LEVEL}" +HOLLOW_PROXY_TEST_LOG_LEVEL="${HOLLOW_PROXY_TEST_LOG_LEVEL:-$TEST_CLUSTER_LOG_LEVEL}" TEST_CLUSTER_DELETE_COLLECTION_WORKERS="${TEST_CLUSTER_DELETE_COLLECTION_WORKERS:---delete-collection-workers=16}" TEST_CLUSTER_MAX_REQUESTS_INFLIGHT="${TEST_CLUSTER_MAX_REQUESTS_INFLIGHT:-}" @@ -96,9 +97,8 @@ CONTROLLER_MANAGER_TEST_ARGS="${KUBEMARK_CONTROLLER_MANAGER_TEST_ARGS:-} ${CONTR SCHEDULER_TEST_ARGS="${KUBEMARK_SCHEDULER_TEST_ARGS:-} ${SCHEDULER_TEST_LOG_LEVEL} ${TEST_CLUSTER_API_CONTENT_TYPE} ${KUBEMARK_MASTER_COMPONENTS_QPS_LIMITS}" # Hollow-node components' test arguments. -KUBELET_TEST_ARGS="--max-pods=100 $TEST_CLUSTER_LOG_LEVEL ${TEST_CLUSTER_API_CONTENT_TYPE}" -KUBEPROXY_TEST_ARGS="${KUBEPROXY_TEST_LOG_LEVEL} ${TEST_CLUSTER_API_CONTENT_TYPE}" -USE_REAL_PROXIER=${USE_REAL_PROXIER:-true} # for hollow-proxy +HOLLOW_KUBELET_TEST_ARGS="${HOLLOW_KUBELET_TEST_ARGS:-} ${HOLLOW_KUBELET_TEST_LOG_LEVEL}" +HOLLOW_PROXY_TEST_ARGS="${HOLLOW_PROXY_TEST_ARGS:-} ${HOLLOW_PROXY_TEST_LOG_LEVEL}" SERVICE_CLUSTER_IP_RANGE="10.0.0.0/16" # formerly PORTAL_NET ALLOCATE_NODE_CIDRS=true diff --git a/cluster/kubemark/pre-existing/config-default.sh b/cluster/kubemark/pre-existing/config-default.sh index de19312971c..5463564edfb 100644 --- a/cluster/kubemark/pre-existing/config-default.sh +++ b/cluster/kubemark/pre-existing/config-default.sh @@ -36,4 +36,3 @@ TEST_CLUSTER_API_CONTENT_TYPE="${TEST_CLUSTER_API_CONTENT_TYPE:-}" KUBELET_TEST_LOG_LEVEL="${KUBELET_TEST_LOG_LEVEL:-}" KUBEPROXY_TEST_LOG_LEVEL="${KUBEPROXY_TEST_LOG_LEVEL:-}" MASTER_NAME="${MASTER_NAME:-}" -USE_REAL_PROXIER="${USE_REAL_PROXIER:-true}" diff --git a/cmd/kubemark/hollow-node.go b/cmd/kubemark/hollow-node.go index fd08daa5e80..8044c1a7d4c 100644 --- a/cmd/kubemark/hollow-node.go +++ b/cmd/kubemark/hollow-node.go @@ -51,14 +51,16 @@ import ( ) type HollowNodeConfig struct { - KubeconfigPath string - KubeletPort int - KubeletReadOnlyPort int - Morph string - NodeName string - ServerPort int - ContentType string - UseRealProxier bool + KubeconfigPath string + KubeletPort int + KubeletReadOnlyPort int + Morph string + NodeName string + ServerPort int + ContentType string + UseRealProxier bool + ProxierSyncPeriod time.Duration + ProxierMinSyncPeriod time.Duration } const ( @@ -66,6 +68,8 @@ const ( podsPerCore = 0 ) +// TODO(#45650): Refactor hollow-node into hollow-kubelet and hollow-proxy +// and make the config driven. var knownMorphs = sets.NewString("kubelet", "proxy") func (c *HollowNodeConfig) addFlags(fs *pflag.FlagSet) { @@ -77,6 +81,8 @@ func (c *HollowNodeConfig) addFlags(fs *pflag.FlagSet) { fs.StringVar(&c.Morph, "morph", "", fmt.Sprintf("Specifies into which Hollow component this binary should morph. Allowed values: %v", knownMorphs.List())) fs.StringVar(&c.ContentType, "kube-api-content-type", "application/vnd.kubernetes.protobuf", "ContentType of requests sent to apiserver.") fs.BoolVar(&c.UseRealProxier, "use-real-proxier", true, "Set to true if you want to use real proxier inside hollow-proxy.") + fs.DurationVar(&c.ProxierSyncPeriod, "proxier-sync-period", 30*time.Second, "Period that proxy rules are refreshed in hollow-proxy.") + fs.DurationVar(&c.ProxierMinSyncPeriod, "proxier-min-sync-period", 0, "Minimum period that proxy rules are refreshed in hollow-proxy.") } func (c *HollowNodeConfig) createClientConfigFromFile() (*restclient.Config, error) { @@ -198,6 +204,8 @@ func run(config *HollowNodeConfig) { eventBroadcaster, recorder, config.UseRealProxier, + config.ProxierSyncPeriod, + config.ProxierMinSyncPeriod, ) if err != nil { glog.Fatalf("Failed to create hollowProxy instance: %v", err) diff --git a/pkg/kubemark/hollow_proxy.go b/pkg/kubemark/hollow_proxy.go index 8ed70fec124..649dea56aa1 100644 --- a/pkg/kubemark/hollow_proxy.go +++ b/pkg/kubemark/hollow_proxy.go @@ -70,6 +70,8 @@ func NewHollowProxyOrDie( broadcaster record.EventBroadcaster, recorder record.EventRecorder, useRealProxier bool, + proxierSyncPeriod time.Duration, + proxierMinSyncPeriod time.Duration, ) (*HollowProxy, error) { // Create proxier and service/endpoint handlers. var proxier proxy.ProxyProvider @@ -83,8 +85,8 @@ func NewHollowProxyOrDie( iptInterface, sysctl, execer, - 30*time.Second, - 5*time.Second, + proxierSyncPeriod, + proxierMinSyncPeriod, false, 0, "10.0.0.0/8", diff --git a/test/kubemark/resources/hollow-node_template.yaml b/test/kubemark/resources/hollow-node_template.yaml index b15c11f30f0..4000eb06b0e 100644 --- a/test/kubemark/resources/hollow-node_template.yaml +++ b/test/kubemark/resources/hollow-node_template.yaml @@ -53,7 +53,7 @@ spec: command: - /bin/sh - -c - - /kubemark --morph=kubelet --name=$(NODE_NAME) --kubeconfig=/kubeconfig/kubelet.kubeconfig $(CONTENT_TYPE) --alsologtostderr {{kubelet_verbosity_level}} 1>>/var/log/kubelet-$(NODE_NAME).log 2>&1 + - /kubemark --morph=kubelet --name=$(NODE_NAME) {{hollow_kubelet_params}} --kubeconfig=/kubeconfig/kubelet.kubeconfig $(CONTENT_TYPE) --alsologtostderr 1>>/var/log/kubelet-$(NODE_NAME).log 2>&1 volumeMounts: - name: kubeconfig-volume mountPath: /kubeconfig @@ -81,7 +81,7 @@ spec: command: - /bin/sh - -c - - /kubemark --morph=proxy --name=$(NODE_NAME) --use-real-proxier={{use_real_proxier}} --kubeconfig=/kubeconfig/kubeproxy.kubeconfig $(CONTENT_TYPE) --alsologtostderr {{kubeproxy_verbosity_level}} 1>>/var/log/kubeproxy-$(NODE_NAME).log 2>&1 + - /kubemark --morph=proxy --name=$(NODE_NAME) {{hollow_proxy_params}} --kubeconfig=/kubeconfig/kubeproxy.kubeconfig $(CONTENT_TYPE) --alsologtostderr 1>>/var/log/kubeproxy-$(NODE_NAME).log 2>&1 volumeMounts: - name: kubeconfig-volume mountPath: /kubeconfig diff --git a/test/kubemark/start-kubemark.sh b/test/kubemark/start-kubemark.sh index 0c794ae37be..4e8b2764b32 100755 --- a/test/kubemark/start-kubemark.sh +++ b/test/kubemark/start-kubemark.sh @@ -405,9 +405,8 @@ current-context: kubemark-context") sed -i'' -e "s'{{kubemark_image_registry}}'${KUBEMARK_IMAGE_REGISTRY}'g" "${RESOURCE_DIRECTORY}/hollow-node.yaml" sed -i'' -e "s/{{kubemark_image_tag}}/${KUBEMARK_IMAGE_TAG}/g" "${RESOURCE_DIRECTORY}/hollow-node.yaml" sed -i'' -e "s/{{master_ip}}/${MASTER_IP}/g" "${RESOURCE_DIRECTORY}/hollow-node.yaml" - sed -i'' -e "s/{{kubelet_verbosity_level}}/${KUBELET_TEST_LOG_LEVEL}/g" "${RESOURCE_DIRECTORY}/hollow-node.yaml" - sed -i'' -e "s/{{kubeproxy_verbosity_level}}/${KUBEPROXY_TEST_LOG_LEVEL}/g" "${RESOURCE_DIRECTORY}/hollow-node.yaml" - sed -i'' -e "s/{{use_real_proxier}}/${USE_REAL_PROXIER}/g" "${RESOURCE_DIRECTORY}/hollow-node.yaml" + sed -i'' -e "s/{{hollow_kubelet_params}}/${HOLLOW_KUBELET_TEST_ARGS}/g" "${RESOURCE_DIRECTORY}/hollow-node.yaml" + sed -i'' -e "s/{{hollow_proxy_params}}/${HOLLOW_PROXY_TEST_ARGS}/g" "${RESOURCE_DIRECTORY}/hollow-node.yaml" sed -i'' -e "s'{{kubemark_mig_config}}'${KUBEMARK_MIG_CONFIG:-}'g" "${RESOURCE_DIRECTORY}/hollow-node.yaml" "${KUBECTL}" create -f "${RESOURCE_DIRECTORY}/hollow-node.yaml" --namespace="kubemark"