From 804a4f558c5b5f9b372a3839c79b8399640da57c Mon Sep 17 00:00:00 2001 From: Shyam Jeedigunta Date: Wed, 17 May 2017 13:34:26 +0200 Subject: [PATCH] Make usage of real proxier in hollow-proxy optional (default=true) --- cluster/kubemark/gce/config-default.sh | 1 + cmd/kubemark/hollow-node.go | 3 + hack/verify-flags/known-flags.txt | 1 + pkg/kubemark/BUILD | 2 + pkg/kubemark/hollow_proxy.go | 75 +++++++++++++------ .../resources/hollow-node_template.yaml | 2 +- test/kubemark/start-kubemark.sh | 1 + 7 files changed, 63 insertions(+), 22 deletions(-) diff --git a/cluster/kubemark/gce/config-default.sh b/cluster/kubemark/gce/config-default.sh index 6e1c6d6e6f1..8aee0259322 100644 --- a/cluster/kubemark/gce/config-default.sh +++ b/cluster/kubemark/gce/config-default.sh @@ -79,6 +79,7 @@ if [ -n "${STORAGE_MEDIA_TYPE:-}" ]; then fi ENABLE_GARBAGE_COLLECTOR=${ENABLE_GARBAGE_COLLECTOR:-true} +USE_REAL_PROXIER=${USE_REAL_PROXIER:-true} # for hollow-proxy KUBELET_TEST_ARGS="--max-pods=100 $TEST_CLUSTER_LOG_LEVEL ${TEST_CLUSTER_API_CONTENT_TYPE}" APISERVER_TEST_ARGS="--runtime-config=extensions/v1beta1 ${API_SERVER_TEST_LOG_LEVEL} ${TEST_CLUSTER_STORAGE_MEDIA_TYPE} ${TEST_CLUSTER_MAX_REQUESTS_INFLIGHT} ${TEST_CLUSTER_DELETE_COLLECTION_WORKERS} --enable-garbage-collector=${ENABLE_GARBAGE_COLLECTOR}" diff --git a/cmd/kubemark/hollow-node.go b/cmd/kubemark/hollow-node.go index 72675407140..0ae673424e5 100644 --- a/cmd/kubemark/hollow-node.go +++ b/cmd/kubemark/hollow-node.go @@ -52,6 +52,7 @@ type HollowNodeConfig struct { NodeName string ServerPort int ContentType string + UseRealProxier bool } const ( @@ -70,6 +71,7 @@ func (c *HollowNodeConfig) addFlags(fs *pflag.FlagSet) { fs.IntVar(&c.ServerPort, "api-server-port", 443, "Port on which API server is listening.") fs.StringVar(&c.Morph, "morph", "", fmt.Sprintf("Specifies into which Hollow component this binary should morph. Allowed values: %v", knownMorphs.List())) fs.StringVar(&c.ContentType, "kube-api-content-type", "application/vnd.kubernetes.protobuf", "ContentType of requests sent to apiserver.") + fs.BoolVar(&c.UseRealProxier, "use-real-proxier", true, "Set to true if you want to use real proxier inside hollow-proxy.") } func (c *HollowNodeConfig) createClientConfigFromFile() (*restclient.Config, error) { @@ -151,6 +153,7 @@ func main() { execer, eventBroadcaster, recorder, + config.UseRealProxier, ) if err != nil { glog.Fatalf("Failed to create hollowProxy instance: %v", err) diff --git a/hack/verify-flags/known-flags.txt b/hack/verify-flags/known-flags.txt index 7ea167c4219..8808408ff4b 100644 --- a/hack/verify-flags/known-flags.txt +++ b/hack/verify-flags/known-flags.txt @@ -703,6 +703,7 @@ upgrade-target use-kubernetes-cluster-service use-kubernetes-version use-legacy-policy-config +use-real-proxier use-service-account-credentials user-whitelist use-service-account-credentials diff --git a/pkg/kubemark/BUILD b/pkg/kubemark/BUILD index 883eb79e96f..d32146b59e6 100644 --- a/pkg/kubemark/BUILD +++ b/pkg/kubemark/BUILD @@ -29,6 +29,8 @@ go_library( "//pkg/kubelet/container/testing:go_default_library", "//pkg/kubelet/dockershim/libdocker:go_default_library", "//pkg/kubelet/types:go_default_library", + "//pkg/proxy:go_default_library", + "//pkg/proxy/config:go_default_library", "//pkg/proxy/iptables:go_default_library", "//pkg/util:go_default_library", "//pkg/util/exec:go_default_library", diff --git a/pkg/kubemark/hollow_proxy.go b/pkg/kubemark/hollow_proxy.go index ed5ce4224ec..01535092f46 100644 --- a/pkg/kubemark/hollow_proxy.go +++ b/pkg/kubemark/hollow_proxy.go @@ -27,7 +27,10 @@ import ( clientv1 "k8s.io/client-go/pkg/api/v1" "k8s.io/client-go/tools/record" proxyapp "k8s.io/kubernetes/cmd/kube-proxy/app" + "k8s.io/kubernetes/pkg/api" clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" + "k8s.io/kubernetes/pkg/proxy" + proxyconfig "k8s.io/kubernetes/pkg/proxy/config" "k8s.io/kubernetes/pkg/proxy/iptables" "k8s.io/kubernetes/pkg/util" utilexec "k8s.io/kubernetes/pkg/util/exec" @@ -42,6 +45,21 @@ type HollowProxy struct { ProxyServer *proxyapp.ProxyServer } +type FakeProxier struct{} + +func (*FakeProxier) Sync() {} +func (*FakeProxier) SyncLoop() { + select {} +} +func (*FakeProxier) OnServiceAdd(service *api.Service) {} +func (*FakeProxier) OnServiceUpdate(oldService, service *api.Service) {} +func (*FakeProxier) OnServiceDelete(service *api.Service) {} +func (*FakeProxier) OnServiceSynced() {} +func (*FakeProxier) OnEndpointsAdd(endpoints *api.Endpoints) {} +func (*FakeProxier) OnEndpointsUpdate(oldEndpoints, endpoints *api.Endpoints) {} +func (*FakeProxier) OnEndpointsDelete(endpoints *api.Endpoints) {} +func (*FakeProxier) OnEndpointsSynced() {} + func NewHollowProxyOrDie( nodeName string, client clientset.Interface, @@ -51,34 +69,49 @@ func NewHollowProxyOrDie( execer utilexec.Interface, broadcaster record.EventBroadcaster, recorder record.EventRecorder, + useRealProxier bool, ) (*HollowProxy, error) { - // Create a proxier with fake iptables underneath it. - proxier, err := iptables.NewProxier( - iptInterface, - sysctl, - execer, - 30*time.Second, - 5*time.Second, - false, - 0, - "10.0.0.0/8", - nodeName, - getNodeIP(client, nodeName), - recorder, - nil, - ) - if err != nil { - return nil, fmt.Errorf("unable to create proxier: %v", err) + // Create proxier and service/endpoint handlers. + var proxier proxy.ProxyProvider + var serviceHandler proxyconfig.ServiceHandler + var endpointsHandler proxyconfig.EndpointsHandler + + if useRealProxier { + // Real proxier with fake iptables, sysctl, etc underneath it. + //var err error + proxierIPTables, err := iptables.NewProxier( + iptInterface, + sysctl, + execer, + 30*time.Second, + 5*time.Second, + false, + 0, + "10.0.0.0/8", + nodeName, + getNodeIP(client, nodeName), + recorder, + nil, + ) + if err != nil { + return nil, fmt.Errorf("unable to create proxier: %v", err) + } + proxier = proxierIPTables + serviceHandler = proxierIPTables + endpointsHandler = proxierIPTables + } else { + proxier = &FakeProxier{} + serviceHandler = &FakeProxier{} + endpointsHandler = &FakeProxier{} } - // Create and start Hollow Proxy + // Create a Hollow Proxy instance. nodeRef := &clientv1.ObjectReference{ Kind: "Node", Name: nodeName, UID: types.UID(nodeName), Namespace: "", } - return &HollowProxy{ ProxyServer: &proxyapp.ProxyServer{ Client: client, @@ -92,8 +125,8 @@ func NewHollowProxyOrDie( OOMScoreAdj: util.Int32Ptr(0), ResourceContainer: "", ConfigSyncPeriod: 30 * time.Second, - ServiceEventHandler: proxier, - EndpointsEventHandler: proxier, + ServiceEventHandler: serviceHandler, + EndpointsEventHandler: endpointsHandler, }, }, nil } diff --git a/test/kubemark/resources/hollow-node_template.yaml b/test/kubemark/resources/hollow-node_template.yaml index a5b629df8d8..505000dccb3 100644 --- a/test/kubemark/resources/hollow-node_template.yaml +++ b/test/kubemark/resources/hollow-node_template.yaml @@ -79,7 +79,7 @@ spec: command: - /bin/sh - -c - - /kubemark --morph=proxy --name=$(NODE_NAME) --kubeconfig=/kubeconfig/kubeproxy.kubeconfig $(CONTENT_TYPE) --alsologtostderr {{kubeproxy_verbosity_level}} 1>>/var/log/kubeproxy-$(NODE_NAME).log 2>&1 + - /kubemark --morph=proxy --name=$(NODE_NAME) --use-real-proxier={{use_real_proxier}} --kubeconfig=/kubeconfig/kubeproxy.kubeconfig $(CONTENT_TYPE) --alsologtostderr {{kubeproxy_verbosity_level}} 1>>/var/log/kubeproxy-$(NODE_NAME).log 2>&1 volumeMounts: - name: kubeconfig-volume mountPath: /kubeconfig diff --git a/test/kubemark/start-kubemark.sh b/test/kubemark/start-kubemark.sh index bce8d3dc316..c7e3e24df6f 100755 --- a/test/kubemark/start-kubemark.sh +++ b/test/kubemark/start-kubemark.sh @@ -322,6 +322,7 @@ current-context: kubemark-context") sed -i'' -e "s/{{master_ip}}/${MASTER_IP}/g" "${RESOURCE_DIRECTORY}/hollow-node.yaml" sed -i'' -e "s/{{kubelet_verbosity_level}}/${KUBELET_TEST_LOG_LEVEL}/g" "${RESOURCE_DIRECTORY}/hollow-node.yaml" sed -i'' -e "s/{{kubeproxy_verbosity_level}}/${KUBEPROXY_TEST_LOG_LEVEL}/g" "${RESOURCE_DIRECTORY}/hollow-node.yaml" + sed -i'' -e "s/{{use_real_proxier}}/${USE_REAL_PROXIER}/g" "${RESOURCE_DIRECTORY}/hollow-node.yaml" "${KUBECTL}" create -f "${RESOURCE_DIRECTORY}/hollow-node.yaml" --namespace="kubemark" echo "Created secrets, configMaps, replication-controllers required for hollow-nodes."