From d14478f27a3bc89a4f6a3939d87121e65cf20246 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lucas=20K=C3=A4ldstr=C3=B6m?= Date: Thu, 6 Jul 2017 20:54:15 +0300 Subject: [PATCH] kubeadm: Make self-hosting work and split out to a phase --- cmd/kubeadm/app/cmd/init.go | 7 +- cmd/kubeadm/app/cmd/phases/phase.go | 1 + cmd/kubeadm/app/cmd/phases/selfhosting.go | 45 +++ cmd/kubeadm/app/constants/constants.go | 5 + .../app/phases/controlplane/manifests.go | 30 -- .../app/phases/controlplane/selfhosted.go | 348 ------------------ .../phases/selfhosting/podspec_mutation.go | 74 ++++ .../app/phases/selfhosting/selfhosting.go | 158 ++++++++ cmd/kubeadm/app/util/apiclient.go | 31 ++ 9 files changed, 320 insertions(+), 379 deletions(-) create mode 100644 cmd/kubeadm/app/cmd/phases/selfhosting.go delete mode 100644 cmd/kubeadm/app/phases/controlplane/selfhosted.go create mode 100644 cmd/kubeadm/app/phases/selfhosting/podspec_mutation.go create mode 100644 cmd/kubeadm/app/phases/selfhosting/selfhosting.go diff --git a/cmd/kubeadm/app/cmd/init.go b/cmd/kubeadm/app/cmd/init.go index f8e87a115e6..8f1e637dd3e 100644 --- a/cmd/kubeadm/app/cmd/init.go +++ b/cmd/kubeadm/app/cmd/init.go @@ -37,6 +37,7 @@ import ( certphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs" controlplanephase "k8s.io/kubernetes/cmd/kubeadm/app/phases/controlplane" kubeconfigphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubeconfig" + selfhostingphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/selfhosting" tokenphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/token" "k8s.io/kubernetes/cmd/kubeadm/app/preflight" kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" @@ -132,6 +133,10 @@ func NewCmdInit(out io.Writer) *cobra.Command { &skipTokenPrint, "skip-token-print", skipTokenPrint, "Skip printing of the default bootstrap token generated by 'kubeadm init'", ) + cmd.PersistentFlags().BoolVar( + &cfg.SelfHosted, "self-hosted", cfg.SelfHosted, + "[experimental] If kubeadm should make this control plane self-hosted", + ) cmd.PersistentFlags().StringVar( &cfg.Token, "token", cfg.Token, @@ -266,7 +271,7 @@ func (i *Init) Run(out io.Writer) error { // Temporary control plane is up, now we create our self hosted control // plane components and remove the static manifests: fmt.Println("[self-hosted] Creating self-hosted control plane...") - if err := controlplanephase.CreateSelfHostedControlPlane(i.cfg, client); err != nil { + if err := selfhostingphase.CreateSelfHostedControlPlane(client); err != nil { return err } } diff --git a/cmd/kubeadm/app/cmd/phases/phase.go b/cmd/kubeadm/app/cmd/phases/phase.go index 0ef3491ff66..62a0b3754c6 100644 --- a/cmd/kubeadm/app/cmd/phases/phase.go +++ b/cmd/kubeadm/app/cmd/phases/phase.go @@ -33,6 +33,7 @@ func NewCmdPhase(out io.Writer) *cobra.Command { cmd.AddCommand(NewCmdKubeConfig(out)) cmd.AddCommand(NewCmdCerts()) cmd.AddCommand(NewCmdPreFlight()) + cmd.AddCommand(NewCmdSelfhosting()) return cmd } diff --git a/cmd/kubeadm/app/cmd/phases/selfhosting.go b/cmd/kubeadm/app/cmd/phases/selfhosting.go new file mode 100644 index 00000000000..9f51ae3c6fe --- /dev/null +++ b/cmd/kubeadm/app/cmd/phases/selfhosting.go @@ -0,0 +1,45 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package phases + +import ( + "github.com/spf13/cobra" + + "k8s.io/kubernetes/cmd/kubeadm/app/phases/selfhosting" + kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" + kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig" +) + +// NewCmdSelfhosting returns the self-hosting Cobra command +func NewCmdSelfhosting() *cobra.Command { + var kubeConfigFile string + cmd := &cobra.Command{ + Use: "selfhosting", + Aliases: []string{"selfhosted"}, + Short: "Make a kubeadm cluster self-hosted.", + Run: func(cmd *cobra.Command, args []string) { + client, err := kubeconfigutil.ClientSetFromFile(kubeConfigFile) + kubeadmutil.CheckErr(err) + + err = selfhosting.CreateSelfHostedControlPlane(client) + kubeadmutil.CheckErr(err) + }, + } + + cmd.Flags().StringVar(&kubeConfigFile, "kubeconfig", "/etc/kubernetes/admin.conf", "The KubeConfig file to use for talking to the cluster") + return cmd +} diff --git a/cmd/kubeadm/app/constants/constants.go b/cmd/kubeadm/app/constants/constants.go index 461c11ea271..3f672e224cd 100644 --- a/cmd/kubeadm/app/constants/constants.go +++ b/cmd/kubeadm/app/constants/constants.go @@ -109,3 +109,8 @@ var ( // MinimumControlPlaneVersion specifies the minimum control plane version kubeadm can deploy MinimumControlPlaneVersion = version.MustParseSemantic("v1.7.0") ) + +// BuildStaticManifestFilepath returns the location on the disk where the Static Pod should be present +func BuildStaticManifestFilepath(componentName string) string { + return filepath.Join(KubernetesDir, ManifestsSubDirName, componentName+".yaml") +} diff --git a/cmd/kubeadm/app/phases/controlplane/manifests.go b/cmd/kubeadm/app/phases/controlplane/manifests.go index c2234f95564..dd3b29f4caf 100644 --- a/cmd/kubeadm/app/phases/controlplane/manifests.go +++ b/cmd/kubeadm/app/phases/controlplane/manifests.go @@ -229,23 +229,6 @@ func pkiVolumeMount() v1.VolumeMount { } } -func flockVolume() v1.Volume { - return v1.Volume{ - Name: "var-lock", - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{Path: "/var/lock"}, - }, - } -} - -func flockVolumeMount() v1.VolumeMount { - return v1.VolumeMount{ - Name: "var-lock", - MountPath: "/var/lock", - ReadOnly: false, - } -} - func k8sVolume() v1.Volume { return v1.Volume{ Name: "k8s", @@ -447,19 +430,6 @@ func getProxyEnvVars() []v1.EnvVar { return envs } -func getSelfHostedAPIServerEnv() []v1.EnvVar { - podIPEnvVar := v1.EnvVar{ - Name: "POD_IP", - ValueFrom: &v1.EnvVarSource{ - FieldRef: &v1.ObjectFieldSelector{ - FieldPath: "status.podIP", - }, - }, - } - - return append(getProxyEnvVars(), podIPEnvVar) -} - // getAuthzParameters gets the authorization-related parameters to the api server // At this point, we can assume the list of authorization modes is valid (due to that it has been validated in the API machinery code already) // If the list is empty; it's defaulted (mostly for unit testing) diff --git a/cmd/kubeadm/app/phases/controlplane/selfhosted.go b/cmd/kubeadm/app/phases/controlplane/selfhosted.go deleted file mode 100644 index 08e4437ad75..00000000000 --- a/cmd/kubeadm/app/phases/controlplane/selfhosted.go +++ /dev/null @@ -1,348 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controlplane - -import ( - "fmt" - "os" - "path/filepath" - "time" - - "k8s.io/api/core/v1" - extensions "k8s.io/api/extensions/v1beta1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/intstr" - "k8s.io/apimachinery/pkg/util/wait" - clientset "k8s.io/client-go/kubernetes" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" - "k8s.io/kubernetes/cmd/kubeadm/app/images" - kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" - "k8s.io/kubernetes/pkg/util/version" -) - -var ( - // maximum unavailable and surge instances per self-hosted component deployment - maxUnavailable = intstr.FromInt(0) - maxSurge = intstr.FromInt(1) -) - -func CreateSelfHostedControlPlane(cfg *kubeadmapi.MasterConfiguration, client *clientset.Clientset) error { - volumes := []v1.Volume{k8sVolume()} - volumeMounts := []v1.VolumeMount{k8sVolumeMount()} - if isCertsVolumeMountNeeded() { - volumes = append(volumes, certsVolume(cfg)) - volumeMounts = append(volumeMounts, certsVolumeMount()) - } - - if isPkiVolumeMountNeeded() { - volumes = append(volumes, pkiVolume()) - volumeMounts = append(volumeMounts, pkiVolumeMount()) - } - - // Need lock for self-hosted - volumes = append(volumes, flockVolume()) - volumeMounts = append(volumeMounts, flockVolumeMount()) - - if err := launchSelfHostedAPIServer(cfg, client, volumes, volumeMounts); err != nil { - return err - } - - if err := launchSelfHostedScheduler(cfg, client, volumes, volumeMounts); err != nil { - return err - } - - if err := launchSelfHostedControllerManager(cfg, client, volumes, volumeMounts); err != nil { - return err - } - - return nil -} - -func launchSelfHostedAPIServer(cfg *kubeadmapi.MasterConfiguration, client *clientset.Clientset, volumes []v1.Volume, volumeMounts []v1.VolumeMount) error { - start := time.Now() - - kubeVersion, err := version.ParseSemantic(cfg.KubernetesVersion) - if err != nil { - return err - } - apiServer := getAPIServerDS(cfg, volumes, volumeMounts, kubeVersion) - if _, err := client.Extensions().DaemonSets(metav1.NamespaceSystem).Create(&apiServer); err != nil { - return fmt.Errorf("failed to create self-hosted %q daemon set [%v]", kubeAPIServer, err) - } - - wait.PollInfinite(kubeadmconstants.APICallRetryInterval, func() (bool, error) { - // TODO: This might be pointless, checking the pods is probably enough. - // It does however get us a count of how many there should be which may be useful - // with HA. - apiDS, err := client.DaemonSets(metav1.NamespaceSystem).Get("self-hosted-"+kubeAPIServer, - metav1.GetOptions{}) - if err != nil { - fmt.Println("[self-hosted] error getting apiserver DaemonSet:", err) - return false, nil - } - fmt.Printf("[self-hosted] %s DaemonSet current=%d, desired=%d\n", - kubeAPIServer, - apiDS.Status.CurrentNumberScheduled, - apiDS.Status.DesiredNumberScheduled) - - if apiDS.Status.CurrentNumberScheduled != apiDS.Status.DesiredNumberScheduled { - return false, nil - } - - return true, nil - }) - - // Wait for self-hosted API server to take ownership - waitForPodsWithLabel(client, "self-hosted-"+kubeAPIServer, true) - - // Remove temporary API server - apiServerStaticManifestPath := buildStaticManifestFilepath(kubeAPIServer) - if err := os.RemoveAll(apiServerStaticManifestPath); err != nil { - return fmt.Errorf("unable to delete temporary API server manifest [%v]", err) - } - - kubeadmutil.WaitForAPI(client) - - fmt.Printf("[self-hosted] self-hosted kube-apiserver ready after %f seconds\n", time.Since(start).Seconds()) - return nil -} - -func launchSelfHostedControllerManager(cfg *kubeadmapi.MasterConfiguration, client *clientset.Clientset, volumes []v1.Volume, volumeMounts []v1.VolumeMount) error { - start := time.Now() - - kubeVersion, err := version.ParseSemantic(cfg.KubernetesVersion) - if err != nil { - return err - } - - ctrlMgr := getControllerManagerDeployment(cfg, volumes, volumeMounts, kubeVersion) - if _, err := client.Extensions().Deployments(metav1.NamespaceSystem).Create(&ctrlMgr); err != nil { - return fmt.Errorf("failed to create self-hosted %q deployment [%v]", kubeControllerManager, err) - } - - waitForPodsWithLabel(client, "self-hosted-"+kubeControllerManager, true) - - ctrlMgrStaticManifestPath := buildStaticManifestFilepath(kubeControllerManager) - if err := os.RemoveAll(ctrlMgrStaticManifestPath); err != nil { - return fmt.Errorf("unable to delete temporary controller manager manifest [%v]", err) - } - - fmt.Printf("[self-hosted] self-hosted kube-controller-manager ready after %f seconds\n", time.Since(start).Seconds()) - return nil - -} - -func launchSelfHostedScheduler(cfg *kubeadmapi.MasterConfiguration, client *clientset.Clientset, volumes []v1.Volume, volumeMounts []v1.VolumeMount) error { - start := time.Now() - scheduler := getSchedulerDeployment(cfg, volumes, volumeMounts) - if _, err := client.Extensions().Deployments(metav1.NamespaceSystem).Create(&scheduler); err != nil { - return fmt.Errorf("failed to create self-hosted %q deployment [%v]", kubeScheduler, err) - } - - waitForPodsWithLabel(client, "self-hosted-"+kubeScheduler, true) - - schedulerStaticManifestPath := buildStaticManifestFilepath(kubeScheduler) - if err := os.RemoveAll(schedulerStaticManifestPath); err != nil { - return fmt.Errorf("unable to delete temporary scheduler manifest [%v]", err) - } - - fmt.Printf("[self-hosted] self-hosted kube-scheduler ready after %f seconds\n", time.Since(start).Seconds()) - return nil -} - -// waitForPodsWithLabel will lookup pods with the given label and wait until they are all -// reporting status as running. -func waitForPodsWithLabel(client *clientset.Clientset, appLabel string, mustBeRunning bool) { - wait.PollInfinite(kubeadmconstants.APICallRetryInterval, func() (bool, error) { - // TODO: Do we need a stronger label link than this? - listOpts := metav1.ListOptions{LabelSelector: fmt.Sprintf("k8s-app=%s", appLabel)} - apiPods, err := client.Pods(metav1.NamespaceSystem).List(listOpts) - if err != nil { - fmt.Printf("[self-hosted] error getting %s pods [%v]\n", appLabel, err) - return false, nil - } - fmt.Printf("[self-hosted] Found %d %s pods\n", len(apiPods.Items), appLabel) - - // TODO: HA - if int32(len(apiPods.Items)) != 1 { - return false, nil - } - for _, pod := range apiPods.Items { - fmt.Printf("[self-hosted] Pod %s status: %s\n", pod.Name, pod.Status.Phase) - if mustBeRunning && pod.Status.Phase != "Running" { - return false, nil - } - } - - return true, nil - }) -} - -// Sources from bootkube templates.go -func getAPIServerDS(cfg *kubeadmapi.MasterConfiguration, volumes []v1.Volume, volumeMounts []v1.VolumeMount, kubeVersion *version.Version) extensions.DaemonSet { - ds := extensions.DaemonSet{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "extensions/v1beta1", - Kind: "DaemonSet", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "self-hosted-" + kubeAPIServer, - Namespace: "kube-system", - Labels: map[string]string{"k8s-app": "self-hosted-" + kubeAPIServer}, - }, - Spec: extensions.DaemonSetSpec{ - Template: v1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "k8s-app": "self-hosted-" + kubeAPIServer, - "component": kubeAPIServer, - "tier": "control-plane", - }, - }, - Spec: v1.PodSpec{ - NodeSelector: map[string]string{kubeadmconstants.LabelNodeRoleMaster: ""}, - HostNetwork: true, - Volumes: volumes, - Containers: []v1.Container{ - { - Name: "self-hosted-" + kubeAPIServer, - Image: images.GetCoreImage(images.KubeAPIServerImage, cfg, kubeadmapi.GlobalEnvParams.HyperkubeImage), - Command: getAPIServerCommand(cfg, true, kubeVersion), - Env: getSelfHostedAPIServerEnv(), - VolumeMounts: volumeMounts, - LivenessProbe: componentProbe(6443, "/healthz", v1.URISchemeHTTPS), - Resources: componentResources("250m"), - }, - }, - Tolerations: []v1.Toleration{kubeadmconstants.MasterToleration}, - DNSPolicy: v1.DNSClusterFirstWithHostNet, - }, - }, - }, - } - return ds -} - -func getControllerManagerDeployment(cfg *kubeadmapi.MasterConfiguration, volumes []v1.Volume, volumeMounts []v1.VolumeMount, kubeVersion *version.Version) extensions.Deployment { - d := extensions.Deployment{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "extensions/v1beta1", - Kind: "Deployment", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "self-hosted-" + kubeControllerManager, - Namespace: "kube-system", - Labels: map[string]string{"k8s-app": "self-hosted-" + kubeControllerManager}, - }, - Spec: extensions.DeploymentSpec{ - // TODO bootkube uses 2 replicas - Strategy: extensions.DeploymentStrategy{ - Type: extensions.RollingUpdateDeploymentStrategyType, - RollingUpdate: &extensions.RollingUpdateDeployment{ - MaxUnavailable: &maxUnavailable, - MaxSurge: &maxSurge, - }, - }, - Template: v1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "k8s-app": "self-hosted-" + kubeControllerManager, - "component": kubeControllerManager, - "tier": "control-plane", - }, - }, - Spec: v1.PodSpec{ - NodeSelector: map[string]string{kubeadmconstants.LabelNodeRoleMaster: ""}, - HostNetwork: true, - Volumes: volumes, - Containers: []v1.Container{ - { - Name: "self-hosted-" + kubeControllerManager, - Image: images.GetCoreImage(images.KubeControllerManagerImage, cfg, kubeadmapi.GlobalEnvParams.HyperkubeImage), - Command: getControllerManagerCommand(cfg, true, kubeVersion), - VolumeMounts: volumeMounts, - LivenessProbe: componentProbe(10252, "/healthz", v1.URISchemeHTTP), - Resources: componentResources("200m"), - Env: getProxyEnvVars(), - }, - }, - Tolerations: []v1.Toleration{kubeadmconstants.MasterToleration}, - DNSPolicy: v1.DNSClusterFirstWithHostNet, - }, - }, - }, - } - return d -} - -func getSchedulerDeployment(cfg *kubeadmapi.MasterConfiguration, volumes []v1.Volume, volumeMounts []v1.VolumeMount) extensions.Deployment { - d := extensions.Deployment{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "extensions/v1beta1", - Kind: "Deployment", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "self-hosted-" + kubeScheduler, - Namespace: "kube-system", - Labels: map[string]string{"k8s-app": "self-hosted-" + kubeScheduler}, - }, - Spec: extensions.DeploymentSpec{ - // TODO bootkube uses 2 replicas - Strategy: extensions.DeploymentStrategy{ - Type: extensions.RollingUpdateDeploymentStrategyType, - RollingUpdate: &extensions.RollingUpdateDeployment{ - MaxUnavailable: &maxUnavailable, - MaxSurge: &maxSurge, - }, - }, - Template: v1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "k8s-app": "self-hosted-" + kubeScheduler, - "component": kubeScheduler, - "tier": "control-plane", - }, - }, - Spec: v1.PodSpec{ - NodeSelector: map[string]string{kubeadmconstants.LabelNodeRoleMaster: ""}, - HostNetwork: true, - Volumes: volumes, - Containers: []v1.Container{ - { - Name: "self-hosted-" + kubeScheduler, - Image: images.GetCoreImage(images.KubeSchedulerImage, cfg, kubeadmapi.GlobalEnvParams.HyperkubeImage), - Command: getSchedulerCommand(cfg, true), - VolumeMounts: volumeMounts, - LivenessProbe: componentProbe(10251, "/healthz", v1.URISchemeHTTP), - Resources: componentResources("100m"), - Env: getProxyEnvVars(), - }, - }, - Tolerations: []v1.Toleration{kubeadmconstants.MasterToleration}, - DNSPolicy: v1.DNSClusterFirstWithHostNet, - }, - }, - }, - } - - return d -} - -func buildStaticManifestFilepath(name string) string { - return filepath.Join(kubeadmapi.GlobalEnvParams.KubernetesDir, kubeadmconstants.ManifestsSubDirName, name+".yaml") -} diff --git a/cmd/kubeadm/app/phases/selfhosting/podspec_mutation.go b/cmd/kubeadm/app/phases/selfhosting/podspec_mutation.go new file mode 100644 index 00000000000..5e5d0f98115 --- /dev/null +++ b/cmd/kubeadm/app/phases/selfhosting/podspec_mutation.go @@ -0,0 +1,74 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package selfhosting + +import ( + "k8s.io/api/core/v1" + kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" +) + +// mutatePodSpec makes a Static Pod-hosted PodSpec suitable for self-hosting +func mutatePodSpec(name string, podSpec *v1.PodSpec) { + mutators := map[string][]func(*v1.PodSpec){ + kubeAPIServer: { + addNodeSelectorToPodSpec, + setMasterTolerationOnPodSpec, + setRightDNSPolicyOnPodSpec, + }, + kubeControllerManager: { + addNodeSelectorToPodSpec, + setMasterTolerationOnPodSpec, + setRightDNSPolicyOnPodSpec, + }, + kubeScheduler: { + addNodeSelectorToPodSpec, + setMasterTolerationOnPodSpec, + setRightDNSPolicyOnPodSpec, + }, + } + + // Get the mutator functions for the component in question, then loop through and execute them + mutatorsForComponent := mutators[name] + for _, mutateFunc := range mutatorsForComponent { + mutateFunc(podSpec) + } +} + +// addNodeSelectorToPodSpec makes Pod require to be scheduled on a node marked with the master label +func addNodeSelectorToPodSpec(podSpec *v1.PodSpec) { + if podSpec.NodeSelector == nil { + podSpec.NodeSelector = map[string]string{kubeadmconstants.LabelNodeRoleMaster: ""} + return + } + + podSpec.NodeSelector[kubeadmconstants.LabelNodeRoleMaster] = "" +} + +// setMasterTolerationOnPodSpec makes the Pod tolerate the master taint +func setMasterTolerationOnPodSpec(podSpec *v1.PodSpec) { + if podSpec.Tolerations == nil { + podSpec.Tolerations = []v1.Toleration{kubeadmconstants.MasterToleration} + return + } + + podSpec.Tolerations = append(podSpec.Tolerations, kubeadmconstants.MasterToleration) +} + +// setRightDNSPolicyOnPodSpec makes sure the self-hosted components can look up things via kube-dns if necessary +func setRightDNSPolicyOnPodSpec(podSpec *v1.PodSpec) { + podSpec.DNSPolicy = v1.DNSClusterFirstWithHostNet +} diff --git a/cmd/kubeadm/app/phases/selfhosting/selfhosting.go b/cmd/kubeadm/app/phases/selfhosting/selfhosting.go new file mode 100644 index 00000000000..36f003bd2fd --- /dev/null +++ b/cmd/kubeadm/app/phases/selfhosting/selfhosting.go @@ -0,0 +1,158 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package selfhosting + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + "time" + + "k8s.io/api/core/v1" + extensions "k8s.io/api/extensions/v1beta1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + kuberuntime "k8s.io/apimachinery/pkg/runtime" + clientset "k8s.io/client-go/kubernetes" + kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" + kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" + "k8s.io/kubernetes/pkg/api" +) + +const ( + kubeAPIServer = "kube-apiserver" + kubeControllerManager = "kube-controller-manager" + kubeScheduler = "kube-scheduler" + + selfHostingPrefix = "self-hosted-" +) + +// CreateSelfHostedControlPlane is responsible for turning a Static Pod-hosted control plane to a self-hosted one +// It achieves that task this way: +// 1. Load the Static Pod specification from disk (from /etc/kubernetes/manifests) +// 2. Extract the PodSpec from that Static Pod specification +// 3. Mutate the PodSpec to be compatible with self-hosting (add the right labels, taints, etc. so it can schedule correctly) +// 4. Build a new DaemonSet object for the self-hosted component in question. Use the above mentioned PodSpec +// 5. Create the DaemonSet resource. Wait until the Pods are running. +// 6. Remove the Static Pod manifest file. The kubelet will stop the original Static Pod-hosted component that was running. +// 7. The self-hosted containers should now step up and take over. +// 8. In order to avoid race conditions, we're still making sure the API /healthz endpoint is healthy +// 9. Do that for the kube-apiserver, kube-controller-manager and kube-scheduler in a loop +func CreateSelfHostedControlPlane(client *clientset.Clientset) error { + + // The sequence here isn't set in stone, but seems to work well to start with the API server + components := []string{kubeAPIServer, kubeControllerManager, kubeScheduler} + + for _, componentName := range components { + start := time.Now() + manifestPath := buildStaticManifestFilepath(componentName) + + // Load the Static Pod file in order to be able to create a self-hosted variant of that file + podSpec, err := loadPodSpecFromFile(manifestPath) + if err != nil { + return err + } + + // Build a DaemonSet object from the loaded PodSpec + ds := buildDaemonSet(componentName, podSpec) + + // Create the DaemonSet in the API Server + if _, err := client.ExtensionsV1beta1().DaemonSets(metav1.NamespaceSystem).Create(ds); err != nil { + if !apierrors.IsAlreadyExists(err) { + return fmt.Errorf("failed to create self-hosted %q daemonset [%v]", componentName, err) + } + + if _, err := client.ExtensionsV1beta1().DaemonSets(metav1.NamespaceSystem).Update(ds); err != nil { + // TODO: We should retry on 409 responses + return fmt.Errorf("failed to update self-hosted %q daemonset [%v]", componentName, err) + } + } + + // Wait for the self-hosted component to come up + kubeadmutil.WaitForPodsWithLabel(client, buildSelfHostedWorkloadLabelQuery(componentName)) + + // Remove the old Static Pod manifest + if err := os.RemoveAll(manifestPath); err != nil { + return fmt.Errorf("unable to delete static pod manifest for %s [%v]", componentName, err) + } + + // Make sure the API is responsive at /healthz + kubeadmutil.WaitForAPI(client) + + fmt.Printf("[self-hosted] self-hosted %s ready after %f seconds\n", componentName, time.Since(start).Seconds()) + } + return nil +} + +// buildDaemonSet is responsible for mutating the PodSpec and return a DaemonSet which is suitable for the self-hosting purporse +func buildDaemonSet(name string, podSpec *v1.PodSpec) *extensions.DaemonSet { + // Mutate the PodSpec so it's suitable for self-hosting + mutatePodSpec(name, podSpec) + + // Return a DaemonSet based on that Spec + return &extensions.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: addSelfHostedPrefix(name), + Namespace: metav1.NamespaceSystem, + Labels: map[string]string{ + "k8s-app": addSelfHostedPrefix(name), + }, + }, + Spec: extensions.DaemonSetSpec{ + Template: v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "k8s-app": addSelfHostedPrefix(name), + }, + }, + Spec: *podSpec, + }, + }, + } +} + +// loadPodSpecFromFile reads and decodes a file containing a specification of a Pod +// TODO: Consider using "k8s.io/kubernetes/pkg/volume/util".LoadPodFromFile(filename string) in the future instead. +func loadPodSpecFromFile(manifestPath string) (*v1.PodSpec, error) { + podBytes, err := ioutil.ReadFile(manifestPath) + if err != nil { + return nil, err + } + + staticPod := &v1.Pod{} + if err := kuberuntime.DecodeInto(api.Codecs.UniversalDecoder(), podBytes, staticPod); err != nil { + return nil, fmt.Errorf("unable to decode static pod %v", err) + } + + return &staticPod.Spec, nil +} + +// buildStaticManifestFilepath returns the location on the disk where the Static Pod should be present +func buildStaticManifestFilepath(componentName string) string { + return filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.ManifestsSubDirName, componentName+".yaml") +} + +// buildSelfHostedWorkloadLabelQuery creates the right query for matching a self-hosted Pod +func buildSelfHostedWorkloadLabelQuery(componentName string) string { + return fmt.Sprintf("k8s-app=%s", addSelfHostedPrefix(componentName)) +} + +// addSelfHostedPrefix adds the self-hosted- prefix to the component name +func addSelfHostedPrefix(componentName string) string { + return fmt.Sprintf("%s%s", selfHostingPrefix, componentName) +} diff --git a/cmd/kubeadm/app/util/apiclient.go b/cmd/kubeadm/app/util/apiclient.go index 2a3c39669ef..e351490a9a9 100644 --- a/cmd/kubeadm/app/util/apiclient.go +++ b/cmd/kubeadm/app/util/apiclient.go @@ -21,12 +21,15 @@ import ( "net/http" "time" + "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" clientset "k8s.io/client-go/kubernetes" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig" ) +// CreateClientAndWaitForAPI takes a path to a kubeconfig file, makes a client of it and waits for the API to be healthy func CreateClientAndWaitForAPI(file string) (*clientset.Clientset, error) { client, err := kubeconfigutil.ClientSetFromFile(file) if err != nil { @@ -39,6 +42,7 @@ func CreateClientAndWaitForAPI(file string) (*clientset.Clientset, error) { return client, nil } +// WaitForAPI waits for the API Server's /healthz endpoint to report "ok" func WaitForAPI(client *clientset.Clientset) { start := time.Now() wait.PollInfinite(kubeadmconstants.APICallRetryInterval, func() (bool, error) { @@ -52,3 +56,30 @@ func WaitForAPI(client *clientset.Clientset) { return true, nil }) } + +// WaitForPodsWithLabel will lookup pods with the given label and wait until they are all +// reporting status as running. +func WaitForPodsWithLabel(client *clientset.Clientset, labelKeyValPair string) { + // TODO: Implement a timeout + // TODO: Implement a verbosity switch + wait.PollInfinite(kubeadmconstants.APICallRetryInterval, func() (bool, error) { + listOpts := metav1.ListOptions{LabelSelector: labelKeyValPair} + apiPods, err := client.CoreV1().Pods(metav1.NamespaceSystem).List(listOpts) + if err != nil { + fmt.Printf("[apiclient] Error getting Pods with label selector %q [%v]\n", labelKeyValPair, err) + return false, nil + } + + if len(apiPods.Items) == 0 { + return false, nil + } + for _, pod := range apiPods.Items { + fmt.Printf("[apiclient] Pod %s status: %s\n", pod.Name, pod.Status.Phase) + if pod.Status.Phase != v1.PodRunning { + return false, nil + } + } + + return true, nil + }) +}