From bbbf530504615d99c2970278116f1e0db0edac59 Mon Sep 17 00:00:00 2001 From: Andrew Rynhard Date: Sun, 6 Aug 2017 15:57:15 -0700 Subject: [PATCH] Refactor addons into multiple packages --- cmd/kubeadm/app/BUILD | 3 +- cmd/kubeadm/app/cmd/BUILD | 3 +- cmd/kubeadm/app/cmd/init.go | 14 +- cmd/kubeadm/app/phases/addons/dns/BUILD | 51 ++++++ .../phases/addons/{addons.go => dns/dns.go} | 76 ++------ cmd/kubeadm/app/phases/addons/dns/dns_test.go | 121 ++++++++++++ .../app/phases/addons/{ => dns}/manifests.go | 88 +-------- .../app/phases/addons/{ => proxy}/BUILD | 19 +- .../app/phases/addons/proxy/manifests.go | 104 +++++++++++ cmd/kubeadm/app/phases/addons/proxy/proxy.go | 172 ++++++++++++++++++ .../{addons_test.go => proxy/proxy_test.go} | 81 +++++++-- cmd/kubeadm/app/phases/apiconfig/BUILD | 2 - .../app/phases/apiconfig/clusterroles.go | 59 ------ .../app/phases/apiconfig/clusterroles_test.go | 66 ------- 14 files changed, 552 insertions(+), 307 deletions(-) create mode 100644 cmd/kubeadm/app/phases/addons/dns/BUILD rename cmd/kubeadm/app/phases/addons/{addons.go => dns/dns.go} (60%) create mode 100644 cmd/kubeadm/app/phases/addons/dns/dns_test.go rename cmd/kubeadm/app/phases/addons/{ => dns}/manifests.go (75%) rename cmd/kubeadm/app/phases/addons/{ => proxy}/BUILD (87%) create mode 100644 cmd/kubeadm/app/phases/addons/proxy/manifests.go create mode 100644 cmd/kubeadm/app/phases/addons/proxy/proxy.go rename cmd/kubeadm/app/phases/addons/{addons_test.go => proxy/proxy_test.go} (55%) diff --git a/cmd/kubeadm/app/BUILD b/cmd/kubeadm/app/BUILD index fb884978fa8..9e170eeef53 100644 --- a/cmd/kubeadm/app/BUILD +++ b/cmd/kubeadm/app/BUILD @@ -33,7 +33,8 @@ filegroup( "//cmd/kubeadm/app/discovery:all-srcs", "//cmd/kubeadm/app/images:all-srcs", "//cmd/kubeadm/app/node:all-srcs", - "//cmd/kubeadm/app/phases/addons:all-srcs", + "//cmd/kubeadm/app/phases/addons/dns:all-srcs", + "//cmd/kubeadm/app/phases/addons/proxy:all-srcs", "//cmd/kubeadm/app/phases/apiconfig:all-srcs", "//cmd/kubeadm/app/phases/bootstraptoken/clusterinfo:all-srcs", "//cmd/kubeadm/app/phases/bootstraptoken/node:all-srcs", diff --git a/cmd/kubeadm/app/cmd/BUILD b/cmd/kubeadm/app/cmd/BUILD index c28fe2504e2..0465655498e 100644 --- a/cmd/kubeadm/app/cmd/BUILD +++ b/cmd/kubeadm/app/cmd/BUILD @@ -25,7 +25,8 @@ go_library( "//cmd/kubeadm/app/cmd/phases:go_default_library", "//cmd/kubeadm/app/constants:go_default_library", "//cmd/kubeadm/app/discovery:go_default_library", - "//cmd/kubeadm/app/phases/addons:go_default_library", + "//cmd/kubeadm/app/phases/addons/dns:go_default_library", + "//cmd/kubeadm/app/phases/addons/proxy:go_default_library", "//cmd/kubeadm/app/phases/apiconfig:go_default_library", "//cmd/kubeadm/app/phases/bootstraptoken/clusterinfo:go_default_library", "//cmd/kubeadm/app/phases/bootstraptoken/node:go_default_library", diff --git a/cmd/kubeadm/app/cmd/init.go b/cmd/kubeadm/app/cmd/init.go index 9933ffd0647..3a295cad482 100644 --- a/cmd/kubeadm/app/cmd/init.go +++ b/cmd/kubeadm/app/cmd/init.go @@ -34,7 +34,8 @@ import ( "k8s.io/kubernetes/cmd/kubeadm/app/cmd/features" cmdphases "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" - addonsphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/addons" + dnsaddonphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/addons/dns" + proxyaddonphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/addons/proxy" apiconfigphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/apiconfig" clusterinfophase "k8s.io/kubernetes/cmd/kubeadm/app/phases/bootstraptoken/clusterinfo" nodebootstraptokenphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/bootstraptoken/node" @@ -289,16 +290,15 @@ func (i *Init) Run(out io.Writer) error { return err } - // Create the necessary ServiceAccounts - if err := apiconfigphase.CreateServiceAccounts(client); err != nil { - return err - } - if err := apiconfigphase.CreateRBACRules(client, k8sVersion); err != nil { return err } - if err := addonsphase.CreateEssentialAddons(i.cfg, client); err != nil { + if err := dnsaddonphase.EnsureDNSAddon(i.cfg, client); err != nil { + return err + } + + if err := proxyaddonphase.EnsureProxyAddon(i.cfg, client); err != nil { return err } diff --git a/cmd/kubeadm/app/phases/addons/dns/BUILD b/cmd/kubeadm/app/phases/addons/dns/BUILD new file mode 100644 index 00000000000..67cc7cd0a4a --- /dev/null +++ b/cmd/kubeadm/app/phases/addons/dns/BUILD @@ -0,0 +1,51 @@ +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) + +load( + "@io_bazel_rules_go//go:def.bzl", + "go_library", + "go_test", +) + +go_test( + name = "go_default_test", + srcs = ["dns_test.go"], + library = ":go_default_library", + tags = ["automanaged"], + deps = ["//cmd/kubeadm/app/util:go_default_library"], +) + +go_library( + name = "go_default_library", + srcs = [ + "dns.go", + "manifests.go", + ], + tags = ["automanaged"], + deps = [ + "//cmd/kubeadm/app/apis/kubeadm:go_default_library", + "//cmd/kubeadm/app/constants:go_default_library", + "//cmd/kubeadm/app/util:go_default_library", + "//pkg/api:go_default_library", + "//vendor/k8s.io/api/core/v1:go_default_library", + "//vendor/k8s.io/api/extensions/v1beta1:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", + "//vendor/k8s.io/client-go/kubernetes:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], +) diff --git a/cmd/kubeadm/app/phases/addons/addons.go b/cmd/kubeadm/app/phases/addons/dns/dns.go similarity index 60% rename from cmd/kubeadm/app/phases/addons/addons.go rename to cmd/kubeadm/app/phases/addons/dns/dns.go index 9accdb5ce36..de9da84c272 100644 --- a/cmd/kubeadm/app/phases/addons/addons.go +++ b/cmd/kubeadm/app/phases/addons/dns/dns.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package addons +package dns import ( "fmt" @@ -30,32 +30,13 @@ import ( kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" - apiclientutil "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" "k8s.io/kubernetes/pkg/api" - "k8s.io/kubernetes/plugin/pkg/scheduler/algorithm" ) -// CreateEssentialAddons creates the kube-proxy and kube-dns addons -func CreateEssentialAddons(cfg *kubeadmapi.MasterConfiguration, client clientset.Interface) error { - proxyConfigMapBytes, err := kubeadmutil.ParseTemplate(KubeProxyConfigMap, struct{ MasterEndpoint string }{ - // Fetch this value from the kubeconfig file - MasterEndpoint: fmt.Sprintf("https://%s:%d", cfg.API.AdvertiseAddress, cfg.API.BindPort), - }) - if err != nil { - return fmt.Errorf("error when parsing kube-proxy configmap template: %v", err) - } - - proxyDaemonSetBytes, err := kubeadmutil.ParseTemplate(KubeProxyDaemonSet, struct{ ImageRepository, Arch, Version, ImageOverride, ClusterCIDR, MasterTaintKey, CloudTaintKey string }{ - ImageRepository: cfg.ImageRepository, - Arch: runtime.GOARCH, - Version: kubeadmutil.KubernetesVersionToImageTag(cfg.KubernetesVersion), - ImageOverride: cfg.UnifiedControlPlaneImage, - ClusterCIDR: getClusterCIDR(cfg.Networking.PodSubnet), - MasterTaintKey: kubeadmconstants.LabelNodeRoleMaster, - CloudTaintKey: algorithm.TaintExternalCloudProvider, - }) - if err != nil { - return fmt.Errorf("error when parsing kube-proxy daemonset template: %v", err) +// EnsureDNSAddon creates the kube-dns addon +func EnsureDNSAddon(cfg *kubeadmapi.MasterConfiguration, client clientset.Interface) error { + if err := CreateServiceAccount(client); err != nil { + return err } dnsDeploymentBytes, err := kubeadmutil.ParseTemplate(KubeDNSDeployment, struct{ ImageRepository, Arch, Version, DNSDomain, MasterTaintKey string }{ @@ -81,48 +62,30 @@ func CreateEssentialAddons(cfg *kubeadmapi.MasterConfiguration, client clientset return fmt.Errorf("error when parsing kube-proxy configmap template: %v", err) } - err = CreateKubeProxyAddon(proxyConfigMapBytes, proxyDaemonSetBytes, client) - if err != nil { - return err - } - fmt.Println("[addons] Applied essential addon: kube-proxy") - - err = CreateKubeDNSAddon(dnsDeploymentBytes, dnsServiceBytes, client) - if err != nil { + if err = createKubeDNSAddon(dnsDeploymentBytes, dnsServiceBytes, client); err != nil { return err } fmt.Println("[addons] Applied essential addon: kube-dns") return nil } -func CreateKubeProxyAddon(configMapBytes, daemonSetbytes []byte, client clientset.Interface) error { - kubeproxyConfigMap := &v1.ConfigMap{} - if err := kuberuntime.DecodeInto(api.Codecs.UniversalDecoder(), configMapBytes, kubeproxyConfigMap); err != nil { - return fmt.Errorf("unable to decode kube-proxy configmap %v", err) +// CreateServiceAccount creates the necessary serviceaccounts that kubeadm uses/might use, if they don't already exist. +func CreateServiceAccount(client clientset.Interface) error { + sa := v1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: kubeadmconstants.KubeDNSServiceAccountName, + Namespace: metav1.NamespaceSystem, + }, } - - if err := apiclientutil.CreateConfigMapIfNotExists(client, kubeproxyConfigMap); err != nil { - return err - } - - kubeproxyDaemonSet := &extensions.DaemonSet{} - if err := kuberuntime.DecodeInto(api.Codecs.UniversalDecoder(), daemonSetbytes, kubeproxyDaemonSet); err != nil { - return fmt.Errorf("unable to decode kube-proxy daemonset %v", err) - } - - if _, err := client.ExtensionsV1beta1().DaemonSets(metav1.NamespaceSystem).Create(kubeproxyDaemonSet); err != nil { + if _, err := client.CoreV1().ServiceAccounts(metav1.NamespaceSystem).Create(&sa); err != nil { if !apierrors.IsAlreadyExists(err) { - return fmt.Errorf("unable to create a new kube-proxy daemonset: %v", err) - } - - if _, err := client.ExtensionsV1beta1().DaemonSets(metav1.NamespaceSystem).Update(kubeproxyDaemonSet); err != nil { - return fmt.Errorf("unable to update the kube-proxy daemonset: %v", err) + return err } } return nil } -func CreateKubeDNSAddon(deploymentBytes, serviceBytes []byte, client clientset.Interface) error { +func createKubeDNSAddon(deploymentBytes, serviceBytes []byte, client clientset.Interface) error { kubednsDeployment := &extensions.Deployment{} if err := kuberuntime.DecodeInto(api.Codecs.UniversalDecoder(), deploymentBytes, kubednsDeployment); err != nil { return fmt.Errorf("unable to decode kube-dns deployment %v", err) @@ -176,10 +139,3 @@ func getDNSIP(client clientset.Interface) (net.IP, error) { } return dnsIP, nil } - -func getClusterCIDR(podsubnet string) string { - if len(podsubnet) == 0 { - return "" - } - return "- --cluster-cidr=" + podsubnet -} diff --git a/cmd/kubeadm/app/phases/addons/dns/dns_test.go b/cmd/kubeadm/app/phases/addons/dns/dns_test.go new file mode 100644 index 00000000000..e65a9d983c1 --- /dev/null +++ b/cmd/kubeadm/app/phases/addons/dns/dns_test.go @@ -0,0 +1,121 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dns + +import ( + "testing" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + clientsetfake "k8s.io/client-go/kubernetes/fake" + core "k8s.io/client-go/testing" + kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" + "k8s.io/kubernetes/pkg/api" +) + +func TestCreateServiceAccount(t *testing.T) { + tests := []struct { + name string + createErr error + expectErr bool + }{ + { + "error-free case", + nil, + false, + }, + { + "duplication errors should be ignored", + apierrors.NewAlreadyExists(api.Resource(""), ""), + false, + }, + { + "unexpected errors should be returned", + apierrors.NewUnauthorized(""), + true, + }, + } + + for _, tc := range tests { + client := clientsetfake.NewSimpleClientset() + if tc.createErr != nil { + client.PrependReactor("create", "serviceaccounts", func(action core.Action) (bool, runtime.Object, error) { + return true, nil, tc.createErr + }) + } + + err := CreateServiceAccount(client) + if tc.expectErr { + if err == nil { + t.Errorf("CreateServiceAccounts(%s) wanted err, got nil", tc.name) + } + continue + } else if !tc.expectErr && err != nil { + t.Errorf("CreateServiceAccounts(%s) returned unexpected err: %v", tc.name, err) + } + + wantResourcesCreated := 1 + if len(client.Actions()) != wantResourcesCreated { + t.Errorf("CreateServiceAccounts(%s) should have made %d actions, but made %d", tc.name, wantResourcesCreated, len(client.Actions())) + } + + for _, action := range client.Actions() { + if action.GetVerb() != "create" || action.GetResource().Resource != "serviceaccounts" { + t.Errorf("CreateServiceAccounts(%s) called [%v %v], but wanted [create serviceaccounts]", + tc.name, action.GetVerb(), action.GetResource().Resource) + } + } + + } +} + +func TestCompileManifests(t *testing.T) { + var tests = []struct { + manifest string + data interface{} + expected bool + }{ + { + manifest: KubeDNSDeployment, + data: struct{ ImageRepository, Arch, Version, DNSDomain, MasterTaintKey string }{ + ImageRepository: "foo", + Arch: "foo", + Version: "foo", + DNSDomain: "foo", + MasterTaintKey: "foo", + }, + expected: true, + }, + { + manifest: KubeDNSService, + data: struct{ DNSIP string }{ + DNSIP: "foo", + }, + expected: true, + }, + } + for _, rt := range tests { + _, actual := kubeadmutil.ParseTemplate(rt.manifest, rt.data) + if (actual == nil) != rt.expected { + t.Errorf( + "failed CompileManifests:\n\texpected: %t\n\t actual: %t", + rt.expected, + (actual == nil), + ) + } + } +} diff --git a/cmd/kubeadm/app/phases/addons/manifests.go b/cmd/kubeadm/app/phases/addons/dns/manifests.go similarity index 75% rename from cmd/kubeadm/app/phases/addons/manifests.go rename to cmd/kubeadm/app/phases/addons/dns/manifests.go index 3b6c6460d8e..866bd474647 100644 --- a/cmd/kubeadm/app/phases/addons/manifests.go +++ b/cmd/kubeadm/app/phases/addons/dns/manifests.go @@ -14,94 +14,13 @@ See the License for the specific language governing permissions and limitations under the License. */ -package addons +package dns const ( - KubeProxyConfigMap = ` -kind: ConfigMap -apiVersion: v1 -metadata: - name: kube-proxy - namespace: kube-system - labels: - app: kube-proxy -data: - kubeconfig.conf: | - apiVersion: v1 - kind: Config - clusters: - - cluster: - certificate-authority: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt - server: {{ .MasterEndpoint }} - name: default - contexts: - - context: - cluster: default - namespace: default - user: default - name: default - current-context: default - users: - - name: default - user: - tokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token -` - - KubeProxyDaemonSet = ` -apiVersion: extensions/v1beta1 -kind: DaemonSet -metadata: - labels: - k8s-app: kube-proxy - name: kube-proxy - namespace: kube-system -spec: - selector: - matchLabels: - k8s-app: kube-proxy - updateStrategy: - type: RollingUpdate - template: - metadata: - labels: - k8s-app: kube-proxy - spec: - containers: - - name: kube-proxy - image: {{ if .ImageOverride }}{{ .ImageOverride }}{{ else }}{{ .ImageRepository }}/kube-proxy-{{ .Arch }}:{{ .Version }}{{ end }} - imagePullPolicy: IfNotPresent - command: - - /usr/local/bin/kube-proxy - - --kubeconfig=/var/lib/kube-proxy/kubeconfig.conf - {{ .ClusterCIDR }} - securityContext: - privileged: true - volumeMounts: - - mountPath: /var/lib/kube-proxy - name: kube-proxy - # TODO: Make this a file hostpath mount - - mountPath: /run/xtables.lock - name: xtables-lock - readOnly: false - hostNetwork: true - serviceAccountName: kube-proxy - tolerations: - - key: {{ .MasterTaintKey }} - effect: NoSchedule - - key: {{ .CloudTaintKey }} - value: "true" - effect: NoSchedule - volumes: - - name: kube-proxy - configMap: - name: kube-proxy - - name: xtables-lock - hostPath: - path: /run/xtables.lock -` - + // KubeDNSVersion is the version of kube-dns to run KubeDNSVersion = "1.14.4" + // KubeDNSDeployment is the kube-dns Deployemnt manifest KubeDNSDeployment = ` apiVersion: extensions/v1beta1 kind: Deployment @@ -268,6 +187,7 @@ spec: - {{ .Arch }} ` + // KubeDNSService is the kube-dns Service manifest KubeDNSService = ` apiVersion: v1 kind: Service diff --git a/cmd/kubeadm/app/phases/addons/BUILD b/cmd/kubeadm/app/phases/addons/proxy/BUILD similarity index 87% rename from cmd/kubeadm/app/phases/addons/BUILD rename to cmd/kubeadm/app/phases/addons/proxy/BUILD index cfef337d446..01c0ac51b57 100644 --- a/cmd/kubeadm/app/phases/addons/BUILD +++ b/cmd/kubeadm/app/phases/addons/proxy/BUILD @@ -6,11 +6,19 @@ load( "go_test", ) +go_test( + name = "go_default_test", + srcs = ["proxy_test.go"], + library = ":go_default_library", + tags = ["automanaged"], + deps = ["//cmd/kubeadm/app/util:go_default_library"], +) + go_library( name = "go_default_library", srcs = [ - "addons.go", "manifests.go", + "proxy.go", ], deps = [ "//cmd/kubeadm/app/apis/kubeadm:go_default_library", @@ -18,9 +26,11 @@ go_library( "//cmd/kubeadm/app/util:go_default_library", "//cmd/kubeadm/app/util/apiclient:go_default_library", "//pkg/api:go_default_library", + "//pkg/util/version:go_default_library", "//plugin/pkg/scheduler/algorithm:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", "//vendor/k8s.io/api/extensions/v1beta1:go_default_library", + "//vendor/k8s.io/api/rbac/v1beta1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", @@ -40,10 +50,3 @@ filegroup( srcs = [":package-srcs"], tags = ["automanaged"], ) - -go_test( - name = "go_default_test", - srcs = ["addons_test.go"], - library = ":go_default_library", - deps = ["//cmd/kubeadm/app/util:go_default_library"], -) diff --git a/cmd/kubeadm/app/phases/addons/proxy/manifests.go b/cmd/kubeadm/app/phases/addons/proxy/manifests.go new file mode 100644 index 00000000000..dfca6bd5e96 --- /dev/null +++ b/cmd/kubeadm/app/phases/addons/proxy/manifests.go @@ -0,0 +1,104 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package proxy + +const ( + // KubeProxyConfigMap is the proxy ConfigMap manifest + KubeProxyConfigMap = ` +kind: ConfigMap +apiVersion: v1 +metadata: + name: kube-proxy + namespace: kube-system + labels: + app: kube-proxy +data: + kubeconfig.conf: | + apiVersion: v1 + kind: Config + clusters: + - cluster: + certificate-authority: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + server: {{ .MasterEndpoint }} + name: default + contexts: + - context: + cluster: default + namespace: default + user: default + name: default + current-context: default + users: + - name: default + user: + tokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token +` + + // KubeProxyDaemonSet is the proxy DaemonSet manifest + KubeProxyDaemonSet = ` +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + labels: + k8s-app: kube-proxy + name: kube-proxy + namespace: kube-system +spec: + selector: + matchLabels: + k8s-app: kube-proxy + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + k8s-app: kube-proxy + spec: + containers: + - name: kube-proxy + image: {{ if .ImageOverride }}{{ .ImageOverride }}{{ else }}{{ .ImageRepository }}/kube-proxy-{{ .Arch }}:{{ .Version }}{{ end }} + imagePullPolicy: IfNotPresent + command: + - /usr/local/bin/kube-proxy + - --kubeconfig=/var/lib/kube-proxy/kubeconfig.conf + {{ .ClusterCIDR }} + securityContext: + privileged: true + volumeMounts: + - mountPath: /var/lib/kube-proxy + name: kube-proxy + # TODO: Make this a file hostpath mount + - mountPath: /run/xtables.lock + name: xtables-lock + readOnly: false + hostNetwork: true + serviceAccountName: kube-proxy + tolerations: + - key: {{ .MasterTaintKey }} + effect: NoSchedule + - key: {{ .CloudTaintKey }} + value: "true" + effect: NoSchedule + volumes: + - name: kube-proxy + configMap: + name: kube-proxy + - name: xtables-lock + hostPath: + path: /run/xtables.lock +` +) diff --git a/cmd/kubeadm/app/phases/addons/proxy/proxy.go b/cmd/kubeadm/app/phases/addons/proxy/proxy.go new file mode 100644 index 00000000000..25a232e5521 --- /dev/null +++ b/cmd/kubeadm/app/phases/addons/proxy/proxy.go @@ -0,0 +1,172 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package proxy + +import ( + "fmt" + "runtime" + + "k8s.io/api/core/v1" + extensions "k8s.io/api/extensions/v1beta1" + rbac "k8s.io/api/rbac/v1beta1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + kuberuntime "k8s.io/apimachinery/pkg/runtime" + clientset "k8s.io/client-go/kubernetes" + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" + kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" + kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" + apiclientutil "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" + "k8s.io/kubernetes/pkg/api" + "k8s.io/kubernetes/pkg/util/version" + k8sversion "k8s.io/kubernetes/pkg/util/version" + "k8s.io/kubernetes/plugin/pkg/scheduler/algorithm" +) + +const ( + // KubeProxyClusterRoleName sets the name for the kube-proxy ClusterRole + // TODO: This k8s-generic, well-known constant should be fetchable from another source, not be in this package + KubeProxyClusterRoleName = "system:node-proxier" +) + +// EnsureProxyAddon creates the kube-proxy and kube-dns addons +func EnsureProxyAddon(cfg *kubeadmapi.MasterConfiguration, client clientset.Interface) error { + if err := CreateServiceAccount(client); err != nil { + return fmt.Errorf("error when creating kube-proxy service account: %v", err) + } + + proxyConfigMapBytes, err := kubeadmutil.ParseTemplate(KubeProxyConfigMap, struct{ MasterEndpoint string }{ + // Fetch this value from the kubeconfig file + MasterEndpoint: fmt.Sprintf("https://%s:%d", cfg.API.AdvertiseAddress, cfg.API.BindPort), + }) + if err != nil { + return fmt.Errorf("error when parsing kube-proxy configmap template: %v", err) + } + + proxyDaemonSetBytes, err := kubeadmutil.ParseTemplate(KubeProxyDaemonSet, struct{ ImageRepository, Arch, Version, ImageOverride, ClusterCIDR, MasterTaintKey, CloudTaintKey string }{ + ImageRepository: cfg.ImageRepository, + Arch: runtime.GOARCH, + Version: kubeadmutil.KubernetesVersionToImageTag(cfg.KubernetesVersion), + ImageOverride: cfg.UnifiedControlPlaneImage, + ClusterCIDR: getClusterCIDR(cfg.Networking.PodSubnet), + MasterTaintKey: kubeadmconstants.LabelNodeRoleMaster, + CloudTaintKey: algorithm.TaintExternalCloudProvider, + }) + if err != nil { + return fmt.Errorf("error when parsing kube-proxy daemonset template: %v", err) + } + + if err = createKubeProxyAddon(proxyConfigMapBytes, proxyDaemonSetBytes, client); err != nil { + return err + } + fmt.Println("[addons] Applied essential addon: kube-proxy") + + k8sVersion, err := version.ParseSemantic(cfg.KubernetesVersion) + if err != nil { + return fmt.Errorf("couldn't parse kubernetes version %q: %v", cfg.KubernetesVersion, err) + } + if err = CreateRBACRules(client, k8sVersion); err != nil { + return fmt.Errorf("error when creating kube-proxy RBAC rules: %v", err) + } + + return nil +} + +// CreateServiceAccount creates the necessary serviceaccounts that kubeadm uses/might use, if they don't already exist. +func CreateServiceAccount(client clientset.Interface) error { + sa := v1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: kubeadmconstants.KubeProxyServiceAccountName, + Namespace: metav1.NamespaceSystem, + }, + } + + if _, err := client.CoreV1().ServiceAccounts(metav1.NamespaceSystem).Create(&sa); err != nil { + if !apierrors.IsAlreadyExists(err) { + return err + } + } + return nil +} + +// CreateRBACRules creates the essential RBAC rules for a minimally set-up cluster +func CreateRBACRules(client clientset.Interface, k8sVersion *k8sversion.Version) error { + if err := createClusterRoleBindings(client); err != nil { + return err + } + return nil +} + +func createKubeProxyAddon(configMapBytes, daemonSetbytes []byte, client clientset.Interface) error { + kubeproxyConfigMap := &v1.ConfigMap{} + if err := kuberuntime.DecodeInto(api.Codecs.UniversalDecoder(), configMapBytes, kubeproxyConfigMap); err != nil { + return fmt.Errorf("unable to decode kube-proxy configmap %v", err) + } + + if _, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Create(kubeproxyConfigMap); err != nil { + if !apierrors.IsAlreadyExists(err) { + return fmt.Errorf("unable to create a new kube-proxy configmap: %v", err) + } + + if _, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Update(kubeproxyConfigMap); err != nil { + return fmt.Errorf("unable to update the kube-proxy configmap: %v", err) + } + } + + kubeproxyDaemonSet := &extensions.DaemonSet{} + if err := kuberuntime.DecodeInto(api.Codecs.UniversalDecoder(), daemonSetbytes, kubeproxyDaemonSet); err != nil { + return fmt.Errorf("unable to decode kube-proxy daemonset %v", err) + } + + if _, err := client.ExtensionsV1beta1().DaemonSets(metav1.NamespaceSystem).Create(kubeproxyDaemonSet); err != nil { + if !apierrors.IsAlreadyExists(err) { + return fmt.Errorf("unable to create a new kube-proxy daemonset: %v", err) + } + + if _, err := client.ExtensionsV1beta1().DaemonSets(metav1.NamespaceSystem).Update(kubeproxyDaemonSet); err != nil { + return fmt.Errorf("unable to update the kube-proxy daemonset: %v", err) + } + } + return nil +} + +func createClusterRoleBindings(client clientset.Interface) error { + return apiclientutil.CreateClusterRoleBindingIfNotExists(client, &rbac.ClusterRoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: "kubeadm:node-proxier", + }, + RoleRef: rbac.RoleRef{ + APIGroup: rbac.GroupName, + Kind: "ClusterRole", + Name: KubeProxyClusterRoleName, + }, + Subjects: []rbac.Subject{ + { + Kind: rbac.ServiceAccountKind, + Name: kubeadmconstants.KubeProxyServiceAccountName, + Namespace: metav1.NamespaceSystem, + }, + }, + }) +} + +func getClusterCIDR(podsubnet string) string { + if len(podsubnet) == 0 { + return "" + } + return "- --cluster-cidr=" + podsubnet +} diff --git a/cmd/kubeadm/app/phases/addons/addons_test.go b/cmd/kubeadm/app/phases/addons/proxy/proxy_test.go similarity index 55% rename from cmd/kubeadm/app/phases/addons/addons_test.go rename to cmd/kubeadm/app/phases/addons/proxy/proxy_test.go index 79cc5c98034..f14b7cf4f1a 100644 --- a/cmd/kubeadm/app/phases/addons/addons_test.go +++ b/cmd/kubeadm/app/phases/addons/proxy/proxy_test.go @@ -14,14 +14,75 @@ See the License for the specific language governing permissions and limitations under the License. */ -package addons +package proxy import ( "testing" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + clientsetfake "k8s.io/client-go/kubernetes/fake" + core "k8s.io/client-go/testing" kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" + "k8s.io/kubernetes/pkg/api" ) +func TestCreateServiceAccount(t *testing.T) { + tests := []struct { + name string + createErr error + expectErr bool + }{ + { + "error-free case", + nil, + false, + }, + { + "duplication errors should be ignored", + apierrors.NewAlreadyExists(api.Resource(""), ""), + false, + }, + { + "unexpected errors should be returned", + apierrors.NewUnauthorized(""), + true, + }, + } + + for _, tc := range tests { + client := clientsetfake.NewSimpleClientset() + if tc.createErr != nil { + client.PrependReactor("create", "serviceaccounts", func(action core.Action) (bool, runtime.Object, error) { + return true, nil, tc.createErr + }) + } + + err := CreateServiceAccount(client) + if tc.expectErr { + if err == nil { + t.Errorf("CreateServiceAccounts(%s) wanted err, got nil", tc.name) + } + continue + } else if !tc.expectErr && err != nil { + t.Errorf("CreateServiceAccounts(%s) returned unexpected err: %v", tc.name, err) + } + + wantResourcesCreated := 1 + if len(client.Actions()) != wantResourcesCreated { + t.Errorf("CreateServiceAccounts(%s) should have made %d actions, but made %d", tc.name, wantResourcesCreated, len(client.Actions())) + } + + for _, action := range client.Actions() { + if action.GetVerb() != "create" || action.GetResource().Resource != "serviceaccounts" { + t.Errorf("CreateServiceAccounts(%s) called [%v %v], but wanted [create serviceaccounts]", + tc.name, action.GetVerb(), action.GetResource().Resource) + } + } + + } +} + func TestGetClusterCIDR(t *testing.T) { emptyClusterCIDR := getClusterCIDR("") if emptyClusterCIDR != "" { @@ -65,24 +126,6 @@ func TestCompileManifests(t *testing.T) { }, expected: true, }, - { - manifest: KubeDNSDeployment, - data: struct{ ImageRepository, Arch, Version, DNSDomain, MasterTaintKey string }{ - ImageRepository: "foo", - Arch: "foo", - Version: "foo", - DNSDomain: "foo", - MasterTaintKey: "foo", - }, - expected: true, - }, - { - manifest: KubeDNSService, - data: struct{ DNSIP string }{ - DNSIP: "foo", - }, - expected: true, - }, } for _, rt := range tests { _, actual := kubeadmutil.ParseTemplate(rt.manifest, rt.data) diff --git a/cmd/kubeadm/app/phases/apiconfig/BUILD b/cmd/kubeadm/app/phases/apiconfig/BUILD index bbd5dfa0f39..8fd10605481 100644 --- a/cmd/kubeadm/app/phases/apiconfig/BUILD +++ b/cmd/kubeadm/app/phases/apiconfig/BUILD @@ -24,9 +24,7 @@ go_library( srcs = ["clusterroles.go"], deps = [ "//cmd/kubeadm/app/constants:go_default_library", - "//cmd/kubeadm/app/util/apiclient:go_default_library", "//pkg/util/version:go_default_library", - "//vendor/k8s.io/api/core/v1:go_default_library", "//vendor/k8s.io/api/rbac/v1beta1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", diff --git a/cmd/kubeadm/app/phases/apiconfig/clusterroles.go b/cmd/kubeadm/app/phases/apiconfig/clusterroles.go index 98730cee86e..a20c9225e42 100644 --- a/cmd/kubeadm/app/phases/apiconfig/clusterroles.go +++ b/cmd/kubeadm/app/phases/apiconfig/clusterroles.go @@ -19,54 +19,16 @@ package apiconfig import ( "fmt" - "k8s.io/api/core/v1" rbac "k8s.io/api/rbac/v1beta1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clientset "k8s.io/client-go/kubernetes" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" - apiclientutil "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" "k8s.io/kubernetes/pkg/util/version" ) -const ( - // KubeProxyClusterRoleName sets the name for the kube-proxy ClusterRole - KubeProxyClusterRoleName = "system:node-proxier" -) - -// CreateServiceAccounts creates the necessary serviceaccounts that kubeadm uses/might use, if they don't already exist. -func CreateServiceAccounts(client clientset.Interface) error { - // TODO: Each ServiceAccount should be created per-addon (decentralized) vs here - serviceAccounts := []v1.ServiceAccount{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: kubeadmconstants.KubeDNSServiceAccountName, - Namespace: metav1.NamespaceSystem, - }, - }, - { - ObjectMeta: metav1.ObjectMeta{ - Name: kubeadmconstants.KubeProxyServiceAccountName, - Namespace: metav1.NamespaceSystem, - }, - }, - } - - for _, sa := range serviceAccounts { - if _, err := client.CoreV1().ServiceAccounts(metav1.NamespaceSystem).Create(&sa); err != nil { - if !apierrors.IsAlreadyExists(err) { - return err - } - } - } - return nil -} - // CreateRBACRules creates the essential RBAC rules for a minimally set-up cluster func CreateRBACRules(client clientset.Interface, k8sVersion *version.Version) error { - if err := createClusterRoleBindings(client); err != nil { - return err - } if err := deletePermissiveNodesBindingWhenUsingNodeAuthorization(client, k8sVersion); err != nil { return fmt.Errorf("failed to remove the permissive 'system:nodes' Group Subject in the 'system:node' ClusterRoleBinding: %v", err) } @@ -75,27 +37,6 @@ func CreateRBACRules(client clientset.Interface, k8sVersion *version.Version) er return nil } -func createClusterRoleBindings(client clientset.Interface) error { - // TODO: This ClusterRoleBinding should be created by the kube-proxy phase, not here - return apiclientutil.CreateClusterRoleBindingIfNotExists(client, &rbac.ClusterRoleBinding{ - ObjectMeta: metav1.ObjectMeta{ - Name: "kubeadm:node-proxier", - }, - RoleRef: rbac.RoleRef{ - APIGroup: rbac.GroupName, - Kind: "ClusterRole", - Name: KubeProxyClusterRoleName, - }, - Subjects: []rbac.Subject{ - { - Kind: rbac.ServiceAccountKind, - Name: kubeadmconstants.KubeProxyServiceAccountName, - Namespace: metav1.NamespaceSystem, - }, - }, - }) -} - func deletePermissiveNodesBindingWhenUsingNodeAuthorization(client clientset.Interface, k8sVersion *version.Version) error { // TODO: When the v1.9 cycle starts (targeting v1.9 at HEAD) and v1.8.0 is the minimum supported version, we can remove this function as the ClusterRoleBinding won't exist diff --git a/cmd/kubeadm/app/phases/apiconfig/clusterroles_test.go b/cmd/kubeadm/app/phases/apiconfig/clusterroles_test.go index 9150c71f5bf..101948f2ec1 100644 --- a/cmd/kubeadm/app/phases/apiconfig/clusterroles_test.go +++ b/cmd/kubeadm/app/phases/apiconfig/clusterroles_test.go @@ -15,69 +15,3 @@ limitations under the License. */ package apiconfig - -import ( - "testing" - - apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/runtime" - clientsetfake "k8s.io/client-go/kubernetes/fake" - core "k8s.io/client-go/testing" - "k8s.io/kubernetes/pkg/api" -) - -func TestCreateServiceAccounts(t *testing.T) { - tests := []struct { - name string - createErr error - expectErr bool - }{ - { - "error-free case", - nil, - false, - }, - { - "duplication errors should be ignored", - apierrors.NewAlreadyExists(api.Resource(""), ""), - false, - }, - { - "unexpected errors should be returned", - apierrors.NewUnauthorized(""), - true, - }, - } - - for _, tc := range tests { - client := clientsetfake.NewSimpleClientset() - if tc.createErr != nil { - client.PrependReactor("create", "serviceaccounts", func(action core.Action) (bool, runtime.Object, error) { - return true, nil, tc.createErr - }) - } - - err := CreateServiceAccounts(client) - if tc.expectErr { - if err == nil { - t.Errorf("CreateServiceAccounts(%s) wanted err, got nil", tc.name) - } - continue - } else if !tc.expectErr && err != nil { - t.Errorf("CreateServiceAccounts(%s) returned unexpected err: %v", tc.name, err) - } - - wantResourcesCreated := 2 - if len(client.Actions()) != wantResourcesCreated { - t.Errorf("CreateServiceAccounts(%s) should have made %d actions, but made %d", tc.name, wantResourcesCreated, len(client.Actions())) - } - - for _, action := range client.Actions() { - if action.GetVerb() != "create" || action.GetResource().Resource != "serviceaccounts" { - t.Errorf("CreateServiceAccounts(%s) called [%v %v], but wanted [create serviceaccounts]", - tc.name, action.GetVerb(), action.GetResource().Resource) - } - } - - } -}