kubeadm: Make self-hosting work and split out to a phase

cmd/kubeadm/app/cmd/init.go

@@ -37,6 +37,7 @@ import (
 	certphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs"
 	controlplanephase "k8s.io/kubernetes/cmd/kubeadm/app/phases/controlplane"
 	kubeconfigphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubeconfig"
+	selfhostingphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/selfhosting"
 	tokenphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/token"
 	"k8s.io/kubernetes/cmd/kubeadm/app/preflight"
 	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
@@ -132,6 +133,10 @@ func NewCmdInit(out io.Writer) *cobra.Command {
 		&skipTokenPrint, "skip-token-print", skipTokenPrint,
 		"Skip printing of the default bootstrap token generated by 'kubeadm init'",
 	)
+	cmd.PersistentFlags().BoolVar(
+		&cfg.SelfHosted, "self-hosted", cfg.SelfHosted,
+		"[experimental] If kubeadm should make this control plane self-hosted",
+	)
 
 	cmd.PersistentFlags().StringVar(
 		&cfg.Token, "token", cfg.Token,
@@ -266,7 +271,7 @@ func (i *Init) Run(out io.Writer) error {
 		// Temporary control plane is up, now we create our self hosted control
 		// plane components and remove the static manifests:
 		fmt.Println("[self-hosted] Creating self-hosted control plane...")
-		if err := controlplanephase.CreateSelfHostedControlPlane(i.cfg, client); err != nil {
+		if err := selfhostingphase.CreateSelfHostedControlPlane(client); err != nil {
 			return err
 		}
 	}

cmd/kubeadm/app/cmd/phases/phase.go

@@ -33,6 +33,7 @@ func NewCmdPhase(out io.Writer) *cobra.Command {
 	cmd.AddCommand(NewCmdKubeConfig(out))
 	cmd.AddCommand(NewCmdCerts())
 	cmd.AddCommand(NewCmdPreFlight())
+	cmd.AddCommand(NewCmdSelfhosting())
 
 	return cmd
 }

cmd/kubeadm/app/cmd/phases/selfhosting.go

@@ -0,0 +1,45 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package phases
+
+import (
+	"github.com/spf13/cobra"
+
+	"k8s.io/kubernetes/cmd/kubeadm/app/phases/selfhosting"
+	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
+	kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig"
+)
+
+// NewCmdSelfhosting returns the self-hosting Cobra command
+func NewCmdSelfhosting() *cobra.Command {
+	var kubeConfigFile string
+	cmd := &cobra.Command{
+		Use:     "selfhosting",
+		Aliases: []string{"selfhosted"},
+		Short:   "Make a kubeadm cluster self-hosted.",
+		Run: func(cmd *cobra.Command, args []string) {
+			client, err := kubeconfigutil.ClientSetFromFile(kubeConfigFile)
+			kubeadmutil.CheckErr(err)
+
+			err = selfhosting.CreateSelfHostedControlPlane(client)
+			kubeadmutil.CheckErr(err)
+		},
+	}
+
+	cmd.Flags().StringVar(&kubeConfigFile, "kubeconfig", "/etc/kubernetes/admin.conf", "The KubeConfig file to use for talking to the cluster")
+	return cmd
+}

cmd/kubeadm/app/constants/constants.go

@@ -109,3 +109,8 @@ var (
 	// MinimumControlPlaneVersion specifies the minimum control plane version kubeadm can deploy
 	MinimumControlPlaneVersion = version.MustParseSemantic("v1.7.0")
 )
+
+// BuildStaticManifestFilepath returns the location on the disk where the Static Pod should be present
+func BuildStaticManifestFilepath(componentName string) string {
+	return filepath.Join(KubernetesDir, ManifestsSubDirName, componentName+".yaml")
+}

cmd/kubeadm/app/phases/controlplane/manifests.go

@@ -229,23 +229,6 @@ func pkiVolumeMount() v1.VolumeMount {
 	}
 }
 
-func flockVolume() v1.Volume {
-	return v1.Volume{
-		Name: "var-lock",
-		VolumeSource: v1.VolumeSource{
-			HostPath: &v1.HostPathVolumeSource{Path: "/var/lock"},
-		},
-	}
-}
-
-func flockVolumeMount() v1.VolumeMount {
-	return v1.VolumeMount{
-		Name:      "var-lock",
-		MountPath: "/var/lock",
-		ReadOnly:  false,
-	}
-}
-
 func k8sVolume() v1.Volume {
 	return v1.Volume{
 		Name: "k8s",
@@ -447,19 +430,6 @@ func getProxyEnvVars() []v1.EnvVar {
 	return envs
 }
 
-func getSelfHostedAPIServerEnv() []v1.EnvVar {
-	podIPEnvVar := v1.EnvVar{
-		Name: "POD_IP",
-		ValueFrom: &v1.EnvVarSource{
-			FieldRef: &v1.ObjectFieldSelector{
-				FieldPath: "status.podIP",
-			},
-		},
-	}
-
-	return append(getProxyEnvVars(), podIPEnvVar)
-}
-
 // getAuthzParameters gets the authorization-related parameters to the api server
 // At this point, we can assume the list of authorization modes is valid (due to that it has been validated in the API machinery code already)
 // If the list is empty; it's defaulted (mostly for unit testing)

cmd/kubeadm/app/phases/controlplane/selfhosted.go

@@ -1,348 +0,0 @@
-/*
-Copyright 2017 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package controlplane
-
-import (
-	"fmt"
-	"os"
-	"path/filepath"
-	"time"
-
-	"k8s.io/api/core/v1"
-	extensions "k8s.io/api/extensions/v1beta1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/util/intstr"
-	"k8s.io/apimachinery/pkg/util/wait"
-	clientset "k8s.io/client-go/kubernetes"
-	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
-	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
-	"k8s.io/kubernetes/cmd/kubeadm/app/images"
-	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
-	"k8s.io/kubernetes/pkg/util/version"
-)
-
-var (
-	// maximum unavailable and surge instances per self-hosted component deployment
-	maxUnavailable = intstr.FromInt(0)
-	maxSurge       = intstr.FromInt(1)
-)
-
-func CreateSelfHostedControlPlane(cfg *kubeadmapi.MasterConfiguration, client *clientset.Clientset) error {
-	volumes := []v1.Volume{k8sVolume()}
-	volumeMounts := []v1.VolumeMount{k8sVolumeMount()}
-	if isCertsVolumeMountNeeded() {
-		volumes = append(volumes, certsVolume(cfg))
-		volumeMounts = append(volumeMounts, certsVolumeMount())
-	}
-
-	if isPkiVolumeMountNeeded() {
-		volumes = append(volumes, pkiVolume())
-		volumeMounts = append(volumeMounts, pkiVolumeMount())
-	}
-
-	// Need lock for self-hosted
-	volumes = append(volumes, flockVolume())
-	volumeMounts = append(volumeMounts, flockVolumeMount())
-
-	if err := launchSelfHostedAPIServer(cfg, client, volumes, volumeMounts); err != nil {
-		return err
-	}
-
-	if err := launchSelfHostedScheduler(cfg, client, volumes, volumeMounts); err != nil {
-		return err
-	}
-
-	if err := launchSelfHostedControllerManager(cfg, client, volumes, volumeMounts); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func launchSelfHostedAPIServer(cfg *kubeadmapi.MasterConfiguration, client *clientset.Clientset, volumes []v1.Volume, volumeMounts []v1.VolumeMount) error {
-	start := time.Now()
-
-	kubeVersion, err := version.ParseSemantic(cfg.KubernetesVersion)
-	if err != nil {
-		return err
-	}
-	apiServer := getAPIServerDS(cfg, volumes, volumeMounts, kubeVersion)
-	if _, err := client.Extensions().DaemonSets(metav1.NamespaceSystem).Create(&apiServer); err != nil {
-		return fmt.Errorf("failed to create self-hosted %q daemon set [%v]", kubeAPIServer, err)
-	}
-
-	wait.PollInfinite(kubeadmconstants.APICallRetryInterval, func() (bool, error) {
-		// TODO: This might be pointless, checking the pods is probably enough.
-		// It does however get us a count of how many there should be which may be useful
-		// with HA.
-		apiDS, err := client.DaemonSets(metav1.NamespaceSystem).Get("self-hosted-"+kubeAPIServer,
-			metav1.GetOptions{})
-		if err != nil {
-			fmt.Println("[self-hosted] error getting apiserver DaemonSet:", err)
-			return false, nil
-		}
-		fmt.Printf("[self-hosted] %s DaemonSet current=%d, desired=%d\n",
-			kubeAPIServer,
-			apiDS.Status.CurrentNumberScheduled,
-			apiDS.Status.DesiredNumberScheduled)
-
-		if apiDS.Status.CurrentNumberScheduled != apiDS.Status.DesiredNumberScheduled {
-			return false, nil
-		}
-
-		return true, nil
-	})
-
-	// Wait for self-hosted API server to take ownership
-	waitForPodsWithLabel(client, "self-hosted-"+kubeAPIServer, true)
-
-	// Remove temporary API server
-	apiServerStaticManifestPath := buildStaticManifestFilepath(kubeAPIServer)
-	if err := os.RemoveAll(apiServerStaticManifestPath); err != nil {
-		return fmt.Errorf("unable to delete temporary API server manifest [%v]", err)
-	}
-
-	kubeadmutil.WaitForAPI(client)
-
-	fmt.Printf("[self-hosted] self-hosted kube-apiserver ready after %f seconds\n", time.Since(start).Seconds())
-	return nil
-}
-
-func launchSelfHostedControllerManager(cfg *kubeadmapi.MasterConfiguration, client *clientset.Clientset, volumes []v1.Volume, volumeMounts []v1.VolumeMount) error {
-	start := time.Now()
-
-	kubeVersion, err := version.ParseSemantic(cfg.KubernetesVersion)
-	if err != nil {
-		return err
-	}
-
-	ctrlMgr := getControllerManagerDeployment(cfg, volumes, volumeMounts, kubeVersion)
-	if _, err := client.Extensions().Deployments(metav1.NamespaceSystem).Create(&ctrlMgr); err != nil {
-		return fmt.Errorf("failed to create self-hosted %q deployment [%v]", kubeControllerManager, err)
-	}
-
-	waitForPodsWithLabel(client, "self-hosted-"+kubeControllerManager, true)
-
-	ctrlMgrStaticManifestPath := buildStaticManifestFilepath(kubeControllerManager)
-	if err := os.RemoveAll(ctrlMgrStaticManifestPath); err != nil {
-		return fmt.Errorf("unable to delete temporary controller manager manifest [%v]", err)
-	}
-
-	fmt.Printf("[self-hosted] self-hosted kube-controller-manager ready after %f seconds\n", time.Since(start).Seconds())
-	return nil
-
-}
-
-func launchSelfHostedScheduler(cfg *kubeadmapi.MasterConfiguration, client *clientset.Clientset, volumes []v1.Volume, volumeMounts []v1.VolumeMount) error {
-	start := time.Now()
-	scheduler := getSchedulerDeployment(cfg, volumes, volumeMounts)
-	if _, err := client.Extensions().Deployments(metav1.NamespaceSystem).Create(&scheduler); err != nil {
-		return fmt.Errorf("failed to create self-hosted %q deployment [%v]", kubeScheduler, err)
-	}
-
-	waitForPodsWithLabel(client, "self-hosted-"+kubeScheduler, true)
-
-	schedulerStaticManifestPath := buildStaticManifestFilepath(kubeScheduler)
-	if err := os.RemoveAll(schedulerStaticManifestPath); err != nil {
-		return fmt.Errorf("unable to delete temporary scheduler manifest [%v]", err)
-	}
-
-	fmt.Printf("[self-hosted] self-hosted kube-scheduler ready after %f seconds\n", time.Since(start).Seconds())
-	return nil
-}
-
-// waitForPodsWithLabel will lookup pods with the given label and wait until they are all
-// reporting status as running.
-func waitForPodsWithLabel(client *clientset.Clientset, appLabel string, mustBeRunning bool) {
-	wait.PollInfinite(kubeadmconstants.APICallRetryInterval, func() (bool, error) {
-		// TODO: Do we need a stronger label link than this?
-		listOpts := metav1.ListOptions{LabelSelector: fmt.Sprintf("k8s-app=%s", appLabel)}
-		apiPods, err := client.Pods(metav1.NamespaceSystem).List(listOpts)
-		if err != nil {
-			fmt.Printf("[self-hosted] error getting %s pods [%v]\n", appLabel, err)
-			return false, nil
-		}
-		fmt.Printf("[self-hosted] Found %d %s pods\n", len(apiPods.Items), appLabel)
-
-		// TODO: HA
-		if int32(len(apiPods.Items)) != 1 {
-			return false, nil
-		}
-		for _, pod := range apiPods.Items {
-			fmt.Printf("[self-hosted] Pod %s status: %s\n", pod.Name, pod.Status.Phase)
-			if mustBeRunning && pod.Status.Phase != "Running" {
-				return false, nil
-			}
-		}
-
-		return true, nil
-	})
-}
-
-// Sources from bootkube templates.go
-func getAPIServerDS(cfg *kubeadmapi.MasterConfiguration, volumes []v1.Volume, volumeMounts []v1.VolumeMount, kubeVersion *version.Version) extensions.DaemonSet {
-	ds := extensions.DaemonSet{
-		TypeMeta: metav1.TypeMeta{
-			APIVersion: "extensions/v1beta1",
-			Kind:       "DaemonSet",
-		},
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      "self-hosted-" + kubeAPIServer,
-			Namespace: "kube-system",
-			Labels:    map[string]string{"k8s-app": "self-hosted-" + kubeAPIServer},
-		},
-		Spec: extensions.DaemonSetSpec{
-			Template: v1.PodTemplateSpec{
-				ObjectMeta: metav1.ObjectMeta{
-					Labels: map[string]string{
-						"k8s-app":   "self-hosted-" + kubeAPIServer,
-						"component": kubeAPIServer,
-						"tier":      "control-plane",
-					},
-				},
-				Spec: v1.PodSpec{
-					NodeSelector: map[string]string{kubeadmconstants.LabelNodeRoleMaster: ""},
-					HostNetwork:  true,
-					Volumes:      volumes,
-					Containers: []v1.Container{
-						{
-							Name:          "self-hosted-" + kubeAPIServer,
-							Image:         images.GetCoreImage(images.KubeAPIServerImage, cfg, kubeadmapi.GlobalEnvParams.HyperkubeImage),
-							Command:       getAPIServerCommand(cfg, true, kubeVersion),
-							Env:           getSelfHostedAPIServerEnv(),
-							VolumeMounts:  volumeMounts,
-							LivenessProbe: componentProbe(6443, "/healthz", v1.URISchemeHTTPS),
-							Resources:     componentResources("250m"),
-						},
-					},
-					Tolerations: []v1.Toleration{kubeadmconstants.MasterToleration},
-					DNSPolicy:   v1.DNSClusterFirstWithHostNet,
-				},
-			},
-		},
-	}
-	return ds
-}
-
-func getControllerManagerDeployment(cfg *kubeadmapi.MasterConfiguration, volumes []v1.Volume, volumeMounts []v1.VolumeMount, kubeVersion *version.Version) extensions.Deployment {
-	d := extensions.Deployment{
-		TypeMeta: metav1.TypeMeta{
-			APIVersion: "extensions/v1beta1",
-			Kind:       "Deployment",
-		},
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      "self-hosted-" + kubeControllerManager,
-			Namespace: "kube-system",
-			Labels:    map[string]string{"k8s-app": "self-hosted-" + kubeControllerManager},
-		},
-		Spec: extensions.DeploymentSpec{
-			// TODO bootkube uses 2 replicas
-			Strategy: extensions.DeploymentStrategy{
-				Type: extensions.RollingUpdateDeploymentStrategyType,
-				RollingUpdate: &extensions.RollingUpdateDeployment{
-					MaxUnavailable: &maxUnavailable,
-					MaxSurge:       &maxSurge,
-				},
-			},
-			Template: v1.PodTemplateSpec{
-				ObjectMeta: metav1.ObjectMeta{
-					Labels: map[string]string{
-						"k8s-app":   "self-hosted-" + kubeControllerManager,
-						"component": kubeControllerManager,
-						"tier":      "control-plane",
-					},
-				},
-				Spec: v1.PodSpec{
-					NodeSelector: map[string]string{kubeadmconstants.LabelNodeRoleMaster: ""},
-					HostNetwork:  true,
-					Volumes:      volumes,
-					Containers: []v1.Container{
-						{
-							Name:          "self-hosted-" + kubeControllerManager,
-							Image:         images.GetCoreImage(images.KubeControllerManagerImage, cfg, kubeadmapi.GlobalEnvParams.HyperkubeImage),
-							Command:       getControllerManagerCommand(cfg, true, kubeVersion),
-							VolumeMounts:  volumeMounts,
-							LivenessProbe: componentProbe(10252, "/healthz", v1.URISchemeHTTP),
-							Resources:     componentResources("200m"),
-							Env:           getProxyEnvVars(),
-						},
-					},
-					Tolerations: []v1.Toleration{kubeadmconstants.MasterToleration},
-					DNSPolicy:   v1.DNSClusterFirstWithHostNet,
-				},
-			},
-		},
-	}
-	return d
-}
-
-func getSchedulerDeployment(cfg *kubeadmapi.MasterConfiguration, volumes []v1.Volume, volumeMounts []v1.VolumeMount) extensions.Deployment {
-	d := extensions.Deployment{
-		TypeMeta: metav1.TypeMeta{
-			APIVersion: "extensions/v1beta1",
-			Kind:       "Deployment",
-		},
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      "self-hosted-" + kubeScheduler,
-			Namespace: "kube-system",
-			Labels:    map[string]string{"k8s-app": "self-hosted-" + kubeScheduler},
-		},
-		Spec: extensions.DeploymentSpec{
-			// TODO bootkube uses 2 replicas
-			Strategy: extensions.DeploymentStrategy{
-				Type: extensions.RollingUpdateDeploymentStrategyType,
-				RollingUpdate: &extensions.RollingUpdateDeployment{
-					MaxUnavailable: &maxUnavailable,
-					MaxSurge:       &maxSurge,
-				},
-			},
-			Template: v1.PodTemplateSpec{
-				ObjectMeta: metav1.ObjectMeta{
-					Labels: map[string]string{
-						"k8s-app":   "self-hosted-" + kubeScheduler,
-						"component": kubeScheduler,
-						"tier":      "control-plane",
-					},
-				},
-				Spec: v1.PodSpec{
-					NodeSelector: map[string]string{kubeadmconstants.LabelNodeRoleMaster: ""},
-					HostNetwork:  true,
-					Volumes:      volumes,
-					Containers: []v1.Container{
-						{
-							Name:          "self-hosted-" + kubeScheduler,
-							Image:         images.GetCoreImage(images.KubeSchedulerImage, cfg, kubeadmapi.GlobalEnvParams.HyperkubeImage),
-							Command:       getSchedulerCommand(cfg, true),
-							VolumeMounts:  volumeMounts,
-							LivenessProbe: componentProbe(10251, "/healthz", v1.URISchemeHTTP),
-							Resources:     componentResources("100m"),
-							Env:           getProxyEnvVars(),
-						},
-					},
-					Tolerations: []v1.Toleration{kubeadmconstants.MasterToleration},
-					DNSPolicy:   v1.DNSClusterFirstWithHostNet,
-				},
-			},
-		},
-	}
-
-	return d
-}
-
-func buildStaticManifestFilepath(name string) string {
-	return filepath.Join(kubeadmapi.GlobalEnvParams.KubernetesDir, kubeadmconstants.ManifestsSubDirName, name+".yaml")
-}

cmd/kubeadm/app/phases/selfhosting/podspec_mutation.go

@@ -0,0 +1,74 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package selfhosting
+
+import (
+	"k8s.io/api/core/v1"
+	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
+)
+
+// mutatePodSpec makes a Static Pod-hosted PodSpec suitable for self-hosting
+func mutatePodSpec(name string, podSpec *v1.PodSpec) {
+	mutators := map[string][]func(*v1.PodSpec){
+		kubeAPIServer: {
+			addNodeSelectorToPodSpec,
+			setMasterTolerationOnPodSpec,
+			setRightDNSPolicyOnPodSpec,
+		},
+		kubeControllerManager: {
+			addNodeSelectorToPodSpec,
+			setMasterTolerationOnPodSpec,
+			setRightDNSPolicyOnPodSpec,
+		},
+		kubeScheduler: {
+			addNodeSelectorToPodSpec,
+			setMasterTolerationOnPodSpec,
+			setRightDNSPolicyOnPodSpec,
+		},
+	}
+
+	// Get the mutator functions for the component in question, then loop through and execute them
+	mutatorsForComponent := mutators[name]
+	for _, mutateFunc := range mutatorsForComponent {
+		mutateFunc(podSpec)
+	}
+}
+
+// addNodeSelectorToPodSpec makes Pod require to be scheduled on a node marked with the master label
+func addNodeSelectorToPodSpec(podSpec *v1.PodSpec) {
+	if podSpec.NodeSelector == nil {
+		podSpec.NodeSelector = map[string]string{kubeadmconstants.LabelNodeRoleMaster: ""}
+		return
+	}
+
+	podSpec.NodeSelector[kubeadmconstants.LabelNodeRoleMaster] = ""
+}
+
+// setMasterTolerationOnPodSpec makes the Pod tolerate the master taint
+func setMasterTolerationOnPodSpec(podSpec *v1.PodSpec) {
+	if podSpec.Tolerations == nil {
+		podSpec.Tolerations = []v1.Toleration{kubeadmconstants.MasterToleration}
+		return
+	}
+
+	podSpec.Tolerations = append(podSpec.Tolerations, kubeadmconstants.MasterToleration)
+}
+
+// setRightDNSPolicyOnPodSpec makes sure the self-hosted components can look up things via kube-dns if necessary
+func setRightDNSPolicyOnPodSpec(podSpec *v1.PodSpec) {
+	podSpec.DNSPolicy = v1.DNSClusterFirstWithHostNet
+}

cmd/kubeadm/app/phases/selfhosting/selfhosting.go

@@ -0,0 +1,158 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package selfhosting
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"time"
+
+	"k8s.io/api/core/v1"
+	extensions "k8s.io/api/extensions/v1beta1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	kuberuntime "k8s.io/apimachinery/pkg/runtime"
+	clientset "k8s.io/client-go/kubernetes"
+	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
+	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
+	"k8s.io/kubernetes/pkg/api"
+)
+
+const (
+	kubeAPIServer         = "kube-apiserver"
+	kubeControllerManager = "kube-controller-manager"
+	kubeScheduler         = "kube-scheduler"
+
+	selfHostingPrefix = "self-hosted-"
+)
+
+// CreateSelfHostedControlPlane is responsible for turning a Static Pod-hosted control plane to a self-hosted one
+// It achieves that task this way:
+// 1. Load the Static Pod specification from disk (from /etc/kubernetes/manifests)
+// 2. Extract the PodSpec from that Static Pod specification
+// 3. Mutate the PodSpec to be compatible with self-hosting (add the right labels, taints, etc. so it can schedule correctly)
+// 4. Build a new DaemonSet object for the self-hosted component in question. Use the above mentioned PodSpec
+// 5. Create the DaemonSet resource. Wait until the Pods are running.
+// 6. Remove the Static Pod manifest file. The kubelet will stop the original Static Pod-hosted component that was running.
+// 7. The self-hosted containers should now step up and take over.
+// 8. In order to avoid race conditions, we're still making sure the API /healthz endpoint is healthy
+// 9. Do that for the kube-apiserver, kube-controller-manager and kube-scheduler in a loop
+func CreateSelfHostedControlPlane(client *clientset.Clientset) error {
+
+	// The sequence here isn't set in stone, but seems to work well to start with the API server
+	components := []string{kubeAPIServer, kubeControllerManager, kubeScheduler}
+
+	for _, componentName := range components {
+		start := time.Now()
+		manifestPath := buildStaticManifestFilepath(componentName)
+
+		// Load the Static Pod file in order to be able to create a self-hosted variant of that file
+		podSpec, err := loadPodSpecFromFile(manifestPath)
+		if err != nil {
+			return err
+		}
+
+		// Build a DaemonSet object from the loaded PodSpec
+		ds := buildDaemonSet(componentName, podSpec)
+
+		// Create the DaemonSet in the API Server
+		if _, err := client.ExtensionsV1beta1().DaemonSets(metav1.NamespaceSystem).Create(ds); err != nil {
+			if !apierrors.IsAlreadyExists(err) {
+				return fmt.Errorf("failed to create self-hosted %q daemonset [%v]", componentName, err)
+			}
+
+			if _, err := client.ExtensionsV1beta1().DaemonSets(metav1.NamespaceSystem).Update(ds); err != nil {
+				// TODO: We should retry on 409 responses
+				return fmt.Errorf("failed to update self-hosted %q daemonset [%v]", componentName, err)
+			}
+		}
+
+		// Wait for the self-hosted component to come up
+		kubeadmutil.WaitForPodsWithLabel(client, buildSelfHostedWorkloadLabelQuery(componentName))
+
+		// Remove the old Static Pod manifest
+		if err := os.RemoveAll(manifestPath); err != nil {
+			return fmt.Errorf("unable to delete static pod manifest for %s [%v]", componentName, err)
+		}
+
+		// Make sure the API is responsive at /healthz
+		kubeadmutil.WaitForAPI(client)
+
+		fmt.Printf("[self-hosted] self-hosted %s ready after %f seconds\n", componentName, time.Since(start).Seconds())
+	}
+	return nil
+}
+
+// buildDaemonSet is responsible for mutating the PodSpec and return a DaemonSet which is suitable for the self-hosting purporse
+func buildDaemonSet(name string, podSpec *v1.PodSpec) *extensions.DaemonSet {
+	// Mutate the PodSpec so it's suitable for self-hosting
+	mutatePodSpec(name, podSpec)
+
+	// Return a DaemonSet based on that Spec
+	return &extensions.DaemonSet{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      addSelfHostedPrefix(name),
+			Namespace: metav1.NamespaceSystem,
+			Labels: map[string]string{
+				"k8s-app": addSelfHostedPrefix(name),
+			},
+		},
+		Spec: extensions.DaemonSetSpec{
+			Template: v1.PodTemplateSpec{
+				ObjectMeta: metav1.ObjectMeta{
+					Labels: map[string]string{
+						"k8s-app": addSelfHostedPrefix(name),
+					},
+				},
+				Spec: *podSpec,
+			},
+		},
+	}
+}
+
+// loadPodSpecFromFile reads and decodes a file containing a specification of a Pod
+// TODO: Consider using "k8s.io/kubernetes/pkg/volume/util".LoadPodFromFile(filename string) in the future instead.
+func loadPodSpecFromFile(manifestPath string) (*v1.PodSpec, error) {
+	podBytes, err := ioutil.ReadFile(manifestPath)
+	if err != nil {
+		return nil, err
+	}
+
+	staticPod := &v1.Pod{}
+	if err := kuberuntime.DecodeInto(api.Codecs.UniversalDecoder(), podBytes, staticPod); err != nil {
+		return nil, fmt.Errorf("unable to decode static pod %v", err)
+	}
+
+	return &staticPod.Spec, nil
+}
+
+// buildStaticManifestFilepath returns the location on the disk where the Static Pod should be present
+func buildStaticManifestFilepath(componentName string) string {
+	return filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.ManifestsSubDirName, componentName+".yaml")
+}
+
+// buildSelfHostedWorkloadLabelQuery creates the right query for matching a self-hosted Pod
+func buildSelfHostedWorkloadLabelQuery(componentName string) string {
+	return fmt.Sprintf("k8s-app=%s", addSelfHostedPrefix(componentName))
+}
+
+// addSelfHostedPrefix adds the self-hosted- prefix to the component name
+func addSelfHostedPrefix(componentName string) string {
+	return fmt.Sprintf("%s%s", selfHostingPrefix, componentName)
+}

cmd/kubeadm/app/util/apiclient.go

@@ -21,12 +21,15 @@ import (
 	"net/http"
 	"time"
 
+	"k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/wait"
 	clientset "k8s.io/client-go/kubernetes"
 	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
 	kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig"
 )
 
+// CreateClientAndWaitForAPI takes a path to a kubeconfig file, makes a client of it and waits for the API to be healthy
 func CreateClientAndWaitForAPI(file string) (*clientset.Clientset, error) {
 	client, err := kubeconfigutil.ClientSetFromFile(file)
 	if err != nil {
@@ -39,6 +42,7 @@ func CreateClientAndWaitForAPI(file string) (*clientset.Clientset, error) {
 	return client, nil
 }
 
+// WaitForAPI waits for the API Server's /healthz endpoint to report "ok"
 func WaitForAPI(client *clientset.Clientset) {
 	start := time.Now()
 	wait.PollInfinite(kubeadmconstants.APICallRetryInterval, func() (bool, error) {
@@ -52,3 +56,30 @@ func WaitForAPI(client *clientset.Clientset) {
 		return true, nil
 	})
 }
+
+// WaitForPodsWithLabel will lookup pods with the given label and wait until they are all
+// reporting status as running.
+func WaitForPodsWithLabel(client *clientset.Clientset, labelKeyValPair string) {
+	// TODO: Implement a timeout
+	// TODO: Implement a verbosity switch
+	wait.PollInfinite(kubeadmconstants.APICallRetryInterval, func() (bool, error) {
+		listOpts := metav1.ListOptions{LabelSelector: labelKeyValPair}
+		apiPods, err := client.CoreV1().Pods(metav1.NamespaceSystem).List(listOpts)
+		if err != nil {
+			fmt.Printf("[apiclient] Error getting Pods with label selector %q [%v]\n", labelKeyValPair, err)
+			return false, nil
+		}
+
+		if len(apiPods.Items) == 0 {
+			return false, nil
+		}
+		for _, pod := range apiPods.Items {
+			fmt.Printf("[apiclient] Pod %s status: %s\n", pod.Name, pod.Status.Phase)
+			if pod.Status.Phase != v1.PodRunning {
+				return false, nil
+			}
+		}
+
+		return true, nil
+	})
+}