From e2893421589adce6aad7c5193085a1cfabbe2edd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lucas=20K=C3=A4ldstr=C3=B6m?= Date: Wed, 1 Feb 2017 23:55:49 +0200 Subject: [PATCH] Move the discovery deployment to a yaml spec --- cmd/kubeadm/app/master/discovery.go | 120 +++++++---------------- cmd/kubeadm/app/master/discovery_test.go | 49 --------- cmd/kubeadm/app/master/templates.go | 62 ++++++++++++ 3 files changed, 96 insertions(+), 135 deletions(-) delete mode 100644 cmd/kubeadm/app/master/discovery_test.go diff --git a/cmd/kubeadm/app/master/discovery.go b/cmd/kubeadm/app/master/discovery.go index 205a3d5f65c..5e9e75afec2 100644 --- a/cmd/kubeadm/app/master/discovery.go +++ b/cmd/kubeadm/app/master/discovery.go @@ -25,27 +25,25 @@ import ( "time" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + kuberuntime "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/wait" certutil "k8s.io/client-go/util/cert" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - kubeadmapiext "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha1" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" + "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/api/v1" extensions "k8s.io/kubernetes/pkg/apis/extensions/v1beta1" "k8s.io/kubernetes/pkg/client/clientset_generated/clientset" ) -type kubeDiscovery struct { - Deployment *extensions.Deployment - Secret *v1.Secret -} - const ( - kubeDiscoveryName = "kube-discovery" kubeDiscoverySecretName = "clusterinfo" + kubeDiscoveryName = "kube-discovery" ) +// TODO: Remove this file as soon as jbeda's token discovery refactoring PR has merged + func encodeKubeDiscoverySecretData(dcfg *kubeadmapi.TokenDiscovery, apicfg kubeadmapi.API, caCert *x509.Certificate) map[string][]byte { var ( data = map[string][]byte{} @@ -66,79 +64,6 @@ func encodeKubeDiscoverySecretData(dcfg *kubeadmapi.TokenDiscovery, apicfg kubea return data } -func newKubeDiscoveryPodSpec(cfg *kubeadmapi.MasterConfiguration) v1.PodSpec { - return v1.PodSpec{ - // We have to use host network namespace, as `HostPort`/`HostIP` are Docker's - // business and CNI support isn't quite there yet (except for kubenet) - // (see https://github.com/kubernetes/kubernetes/issues/31307) - // TODO update this when #31307 is resolved - HostNetwork: true, - SecurityContext: &v1.PodSecurityContext{}, - Containers: []v1.Container{{ - Name: kubeDiscoveryName, - Image: kubeadmapi.GlobalEnvParams.DiscoveryImage, - Command: []string{"/usr/local/bin/kube-discovery"}, - VolumeMounts: []v1.VolumeMount{{ - Name: kubeDiscoverySecretName, - MountPath: "/tmp/secret", // TODO use a shared constant - ReadOnly: true, - }}, - Ports: []v1.ContainerPort{ - // TODO when CNI issue (#31307) is resolved, we should consider adding - // `HostIP: s.API.AdvertiseAddrs[0]`, if there is only one address` - {Name: "http", ContainerPort: kubeadmapiext.DefaultDiscoveryBindPort, HostPort: kubeadmutil.DiscoveryPort(cfg.Discovery.Token)}, - }, - SecurityContext: &v1.SecurityContext{ - SELinuxOptions: &v1.SELinuxOptions{ - // TODO: This implies our discovery container is not being restricted by - // SELinux. This is not optimal and would be nice to adjust in future - // so it can read /tmp/secret, but for now this avoids recommending - // setenforce 0 system-wide. - Type: "spc_t", - }, - }, - }}, - Volumes: []v1.Volume{{ - Name: kubeDiscoverySecretName, - VolumeSource: v1.VolumeSource{ - Secret: &v1.SecretVolumeSource{SecretName: kubeDiscoverySecretName}, - }}, - }, - Affinity: &v1.Affinity{ - NodeAffinity: &v1.NodeAffinity{ - RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ - NodeSelectorTerms: []v1.NodeSelectorTerm{ - { - MatchExpressions: []v1.NodeSelectorRequirement{ - { - Key: "beta.kubernetes.io/arch", - Operator: v1.NodeSelectorOpIn, - Values: []string{runtime.GOARCH}, - }, - }, - }, - }, - }, - }, - }, - } -} - -func newKubeDiscovery(cfg *kubeadmapi.MasterConfiguration, caCert *x509.Certificate) kubeDiscovery { - kd := kubeDiscovery{ - Deployment: NewDeployment(kubeDiscoveryName, 1, newKubeDiscoveryPodSpec(cfg)), - Secret: &v1.Secret{ - ObjectMeta: metav1.ObjectMeta{Name: kubeDiscoverySecretName}, - Type: v1.SecretTypeOpaque, - Data: encodeKubeDiscoverySecretData(cfg.Discovery.Token, cfg.API, caCert), - }, - } - - SetMasterTaintTolerations(&kd.Deployment.Spec.Template.ObjectMeta) - - return kd -} - func CreateDiscoveryDeploymentAndSecret(cfg *kubeadmapi.MasterConfiguration, client *clientset.Clientset) error { caCertificatePath := path.Join(kubeadmapi.GlobalEnvParams.HostPKIPath, kubeadmconstants.CACertName) caCerts, err := certutil.CertsFromFile(caCertificatePath) @@ -150,19 +75,23 @@ func CreateDiscoveryDeploymentAndSecret(cfg *kubeadmapi.MasterConfiguration, cli // TODO: Support multiple certs here in order to be able to rotate certs caCert := caCerts[0] - kd := newKubeDiscovery(cfg, caCert) - - if _, err := client.Extensions().Deployments(metav1.NamespaceSystem).Create(kd.Deployment); err != nil { - return fmt.Errorf("failed to create %q deployment [%v]", kubeDiscoveryName, err) + secret := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: kubeDiscoverySecretName}, + Type: v1.SecretTypeOpaque, + Data: encodeKubeDiscoverySecretData(cfg.Discovery.Token, cfg.API, caCert), } - if _, err := client.Secrets(metav1.NamespaceSystem).Create(kd.Secret); err != nil { + if _, err := client.Secrets(metav1.NamespaceSystem).Create(secret); err != nil { return fmt.Errorf("failed to create %q secret [%v]", kubeDiscoverySecretName, err) } + if err := createDiscoveryDeployment(client); err != nil { + return err + } + fmt.Println("[token-discovery] Created the kube-discovery deployment, waiting for it to become ready") start := time.Now() - wait.PollInfinite(apiCallRetryInterval, func() (bool, error) { + wait.PollInfinite(kubeadmconstants.APICallRetryInterval, func() (bool, error) { d, err := client.Extensions().Deployments(metav1.NamespaceSystem).Get(kubeDiscoveryName, metav1.GetOptions{}) if err != nil { return false, nil @@ -176,3 +105,22 @@ func CreateDiscoveryDeploymentAndSecret(cfg *kubeadmapi.MasterConfiguration, cli return nil } + +func createDiscoveryDeployment(client *clientset.Clientset) error { + discoveryBytes, err := kubeadmutil.ParseTemplate(KubeDiscoveryDeployment, struct{ ImageRepository, Arch string }{ + ImageRepository: kubeadmapi.GlobalEnvParams.RepositoryPrefix, + Arch: runtime.GOARCH, + }) + if err != nil { + return fmt.Errorf("error when parsing kube-discovery template: %v", err) + } + + discoveryDeployment := &extensions.Deployment{} + if err := kuberuntime.DecodeInto(api.Codecs.UniversalDecoder(), discoveryBytes, discoveryDeployment); err != nil { + return fmt.Errorf("unable to decode kube-discovery deployment %v", err) + } + if _, err := client.ExtensionsV1beta1().Deployments(metav1.NamespaceSystem).Create(discoveryDeployment); err != nil { + return fmt.Errorf("unable to create a new discovery deployment: %v", err) + } + return nil +} diff --git a/cmd/kubeadm/app/master/discovery_test.go b/cmd/kubeadm/app/master/discovery_test.go deleted file mode 100644 index 1f6c16bbc12..00000000000 --- a/cmd/kubeadm/app/master/discovery_test.go +++ /dev/null @@ -1,49 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package master - -import ( - "crypto/x509" - "testing" - - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" -) - -func TestNewKubeDiscovery(t *testing.T) { - var tests = []struct { - cfg *kubeadmapi.MasterConfiguration - caCert *x509.Certificate - expected bool - }{ - { - cfg: &kubeadmapi.MasterConfiguration{ - API: kubeadmapi.API{Port: 123, AdvertiseAddresses: []string{"10.0.0.1"}}, - Networking: kubeadmapi.Networking{ServiceSubnet: "10.0.0.1/1"}, - Discovery: kubeadmapi.Discovery{Token: &kubeadmapi.TokenDiscovery{}}, - }, - caCert: &x509.Certificate{}, - }, - } - for _, rt := range tests { - actual := newKubeDiscovery(rt.cfg, rt.caCert) - if actual.Deployment == nil || actual.Secret == nil { - t.Errorf( - "failed newKubeDiscovery, kubeDiscovery was nil", - ) - } - } -} diff --git a/cmd/kubeadm/app/master/templates.go b/cmd/kubeadm/app/master/templates.go index 5b2e88301cf..9c51ed88cbc 100644 --- a/cmd/kubeadm/app/master/templates.go +++ b/cmd/kubeadm/app/master/templates.go @@ -39,5 +39,67 @@ spec: - image: {{ .ImageRepository }}/pause-{{ .Arch }}:3.0 name: dummy hostNetwork: true +` + KubeDiscoveryDeployment = ` +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + labels: + k8s-app: kube-discovery + kubernetes.io/cluster-service: "true" + name: kube-discovery + namespace: kube-system +spec: + replicas: 1 + selector: + matchLabels: + k8s-app: kube-discovery + kubernetes.io/cluster-service: "true" + template: + metadata: + labels: + k8s-app: kube-discovery + # TODO: I guess we can remove all these cluster-service labels... + kubernetes.io/cluster-service: "true" + annotations: + # TODO: Move this to the beta tolerations field below as soon as the Tolerations field exists in PodSpec + scheduler.alpha.kubernetes.io/tolerations: '[{"key":"dedicated","value":"master","effect":"NoSchedule"}]' + spec: + containers: + - name: kube-discovery + image: {{ .ImageRepository }}/kube-discovery-{{ .Arch }}:1.0 + imagePullPolicy: IfNotPresent + command: + - /usr/local/bin/kube-discovery + ports: + - containerPort: 9898 + hostPort: 9898 + name: http + volumeMounts: + - mountPath: /tmp/secret + name: clusterinfo + readOnly: true + hostNetwork: true + # tolerations: + # - key: dedicated + # value: master + # effect: NoSchedule + securityContext: + seLinuxOptions: + type: spc_t + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: beta.kubernetes.io/arch + operator: In + values: + - {{ .Arch }} + volumes: + - name: clusterinfo + secret: + defaultMode: 420 + secretName: clusterinfo ` )