From 750cdb5bc2f46e844de7bcb23417e1caed06d92c Mon Sep 17 00:00:00 2001 From: Devan Goodwin Date: Wed, 7 Dec 2016 08:51:49 -0400 Subject: [PATCH 1/3] kubeadm: first pass at self-hosted master components. --- cmd/kubeadm/app/cmd/init.go | 178 ++++++++++++-- cmd/kubeadm/app/master/BUILD | 1 + cmd/kubeadm/app/master/apiclient.go | 62 ++--- cmd/kubeadm/app/master/manifests.go | 13 +- cmd/kubeadm/app/master/selfhosted.go | 334 +++++++++++++++++++++++++++ 5 files changed, 539 insertions(+), 49 deletions(-) create mode 100644 cmd/kubeadm/app/master/selfhosted.go diff --git a/cmd/kubeadm/app/cmd/init.go b/cmd/kubeadm/app/cmd/init.go index 124f15e17e3..cdb7a391aaf 100644 --- a/cmd/kubeadm/app/cmd/init.go +++ b/cmd/kubeadm/app/cmd/init.go @@ -21,10 +21,13 @@ import ( "io" "io/ioutil" "path" + "strconv" "github.com/renstrom/dedent" "github.com/spf13/cobra" + "k8s.io/apimachinery/pkg/runtime" + netutil "k8s.io/apimachinery/pkg/util/net" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmapiext "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha1" "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation" @@ -32,12 +35,8 @@ import ( "k8s.io/kubernetes/cmd/kubeadm/app/discovery" kubemaster "k8s.io/kubernetes/cmd/kubeadm/app/master" "k8s.io/kubernetes/cmd/kubeadm/app/phases/apiconfig" - certphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs" kubeconfigphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubeconfig" - - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/kubernetes/cmd/kubeadm/app/preflight" kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" "k8s.io/kubernetes/pkg/api" @@ -55,6 +54,9 @@ var ( kubeadm join --discovery %s `) + deploymentStaticPod = "static-pods" + deploymentSelfHosted = "self-hosted" + deploymentTypes = []string{deploymentStaticPod, deploymentSelfHosted} ) // NewCmdInit returns "kubeadm init" command. @@ -66,13 +68,14 @@ func NewCmdInit(out io.Writer) *cobra.Command { var cfgPath string var skipPreFlight bool + var deploymentType string // static pods, self-hosted, etc. cmd := &cobra.Command{ Use: "init", Short: "Run this in order to set up the Kubernetes master", Run: func(cmd *cobra.Command, args []string) { - i, err := NewInit(cfgPath, &cfg, skipPreFlight) + i, err := NewInit(cfgPath, &cfg, skipPreFlight, deploymentType) kubeadmutil.CheckErr(err) - kubeadmutil.CheckErr(i.Validate()) + kubeadmutil.CheckErr(Validate(i.Cfg())) kubeadmutil.CheckErr(i.Run(out)) }, } @@ -123,14 +126,15 @@ func NewCmdInit(out io.Writer) *cobra.Command { "The discovery method kubeadm will use for connecting nodes to the master", ) + cmd.PersistentFlags().StringVar( + &deploymentType, "deployment", deploymentType, + fmt.Sprintf("specify a deployment type from %v", deploymentTypes), + ) + return cmd } -type Init struct { - cfg *kubeadmapi.MasterConfiguration -} - -func NewInit(cfgPath string, cfg *kubeadmapi.MasterConfiguration, skipPreFlight bool) (*Init, error) { +func NewInit(cfgPath string, cfg *kubeadmapi.MasterConfiguration, skipPreFlight bool, deploymentType string) (Init, error) { fmt.Println("[kubeadm] WARNING: kubeadm is in alpha, please do not use it for production clusters.") @@ -169,15 +173,64 @@ func NewInit(cfgPath string, cfg *kubeadmapi.MasterConfiguration, skipPreFlight // Try to start the kubelet service in case it's inactive preflight.TryStartKubelet() - return &Init{cfg: cfg}, nil + // validate version argument + ver, err := kubeadmutil.KubernetesReleaseVersion(cfg.KubernetesVersion) + if err != nil { + if cfg.KubernetesVersion != kubeadmapiext.DefaultKubernetesVersion { + return nil, err + } else { + ver = kubeadmapiext.DefaultKubernetesFallbackVersion + } + } + cfg.KubernetesVersion = ver + fmt.Println("[init] Using Kubernetes version:", ver) + fmt.Println("[init] Using Authorization mode:", cfg.AuthorizationMode) + + // Warn about the limitations with the current cloudprovider solution. + if cfg.CloudProvider != "" { + fmt.Println("WARNING: For cloudprovider integrations to work --cloud-provider must be set for all kubelets in the cluster.") + fmt.Println("\t(/etc/systemd/system/kubelet.service.d/10-kubeadm.conf should be edited for this purpose)") + } + + var deploymentTypeValid bool + for _, supportedDT := range deploymentTypes { + if deploymentType == supportedDT { + deploymentTypeValid = true + } + } + if !deploymentTypeValid { + return nil, fmt.Errorf("%s is not a valid deployment type, you can use any of %v or leave unset to accept the default", deploymentType, deploymentTypes) + } + if deploymentType == deploymentSelfHosted { + fmt.Println("[init] Creating self-hosted Kubernetes deployment...") + return &SelfHostedInit{cfg: cfg}, nil + } + + fmt.Println("[init] Creating static pod Kubernetes deployment...") + return &StaticPodInit{cfg: cfg}, nil } -func (i *Init) Validate() error { - return validation.ValidateMasterConfiguration(i.cfg).ToAggregate() +func Validate(cfg *kubeadmapi.MasterConfiguration) error { + return validation.ValidateMasterConfiguration(cfg).ToAggregate() +} + +// Init structs define implementations of the cluster setup for each supported +// delpoyment type. +type Init interface { + Cfg() *kubeadmapi.MasterConfiguration + Run(out io.Writer) error +} + +type StaticPodInit struct { + cfg *kubeadmapi.MasterConfiguration +} + +func (spi *StaticPodInit) Cfg() *kubeadmapi.MasterConfiguration { + return spi.cfg } // Run executes master node provisioning, including certificates, needed static pod manifests, etc. -func (i *Init) Run(out io.Writer) error { +func (i *StaticPodInit) Run(out io.Writer) error { // PHASE 1: Generate certificates caCert, err := certphase.CreatePKIAssets(i.cfg, kubeadmapi.GlobalEnvParams.HostPKIPath) @@ -254,6 +307,101 @@ func (i *Init) Run(out io.Writer) error { return nil } +// SelfHostedInit initializes a self-hosted cluster. +type SelfHostedInit struct { + cfg *kubeadmapi.MasterConfiguration +} + +func (spi *SelfHostedInit) Cfg() *kubeadmapi.MasterConfiguration { + return spi.cfg +} + +// Run executes master node provisioning, including certificates, needed pod manifests, etc. +func (i *SelfHostedInit) Run(out io.Writer) error { + // Validate token if any, otherwise generate + if i.cfg.Discovery.Token != nil { + if i.cfg.Discovery.Token.ID != "" && i.cfg.Discovery.Token.Secret != "" { + fmt.Printf("[token-discovery] A token has been provided, validating [%s]\n", kubeadmutil.BearerToken(i.cfg.Discovery.Token)) + if valid, err := kubeadmutil.ValidateToken(i.cfg.Discovery.Token); valid == false { + return err + } + } else { + fmt.Println("[token-discovery] A token has not been provided, generating one") + if err := kubeadmutil.GenerateToken(i.cfg.Discovery.Token); err != nil { + return err + } + } + + // Make sure there is at least one address + if len(i.cfg.Discovery.Token.Addresses) == 0 { + ip, err := netutil.ChooseHostInterface() + if err != nil { + return err + } + i.cfg.Discovery.Token.Addresses = []string{ip.String() + ":" + strconv.Itoa(kubeadmapiext.DefaultDiscoveryBindPort)} + } + + if err := kubemaster.CreateTokenAuthFile(kubeadmutil.BearerToken(i.cfg.Discovery.Token)); err != nil { + return err + } + } + + // PHASE 1: Generate certificates + caCert, err := certphase.CreatePKIAssets(i.cfg, kubeadmapi.GlobalEnvParams.HostPKIPath) + if err != nil { + return err + } + + // PHASE 2: Generate kubeconfig files for the admin and the kubelet + + // TODO this is not great, but there is only one address we can use here + // so we'll pick the first one, there is much of chance to have an empty + // slice by the time this gets called + masterEndpoint := fmt.Sprintf("https://%s:%d", i.cfg.API.AdvertiseAddresses[0], i.cfg.API.Port) + err = kubeconfigphase.CreateAdminAndKubeletKubeConfig(masterEndpoint, kubeadmapi.GlobalEnvParams.HostPKIPath, kubeadmapi.GlobalEnvParams.KubernetesDir) + if err != nil { + return err + } + + // Phase 3: Bootstrap the control plane + if err := kubemaster.WriteStaticPodManifests(i.cfg); err != nil { + return err + } + + client, err := kubemaster.CreateClientAndWaitForAPI(path.Join(kubeadmapi.GlobalEnvParams.KubernetesDir, kubeconfigphase.AdminKubeConfigFileName)) + if err != nil { + return err + } + + if err := kubemaster.UpdateMasterRoleLabelsAndTaints(client, false); err != nil { + return err + } + + // Temporary control plane is up, now we create our self hosted control + // plane components and remove the static manifests: + fmt.Println("[init] Creating self-hosted control plane...") + if err := kubemaster.CreateSelfHostedControlPlane(i.cfg, client); err != nil { + return err + } + + if i.cfg.Discovery.Token != nil { + fmt.Printf("[token-discovery] Using token: %s\n", kubeadmutil.BearerToken(i.cfg.Discovery.Token)) + if err := kubemaster.CreateDiscoveryDeploymentAndSecret(i.cfg, client, caCert); err != nil { + return err + } + if err := kubeadmutil.UpdateOrCreateToken(client, i.cfg.Discovery.Token, kubeadmutil.DefaultTokenDuration); err != nil { + return err + } + } + + if err := kubemaster.CreateEssentialAddons(i.cfg, client); err != nil { + return err + } + + fmt.Fprintf(out, initDoneMsgf, generateJoinArgs(i.cfg)) + return nil +} + // generateJoinArgs generates kubeadm join arguments func generateJoinArgs(cfg *kubeadmapi.MasterConfiguration) string { return discovery.NewDiscoveryValue(&cfg.Discovery).String() diff --git a/cmd/kubeadm/app/master/BUILD b/cmd/kubeadm/app/master/BUILD index d2ffeb85f39..af147376b46 100644 --- a/cmd/kubeadm/app/master/BUILD +++ b/cmd/kubeadm/app/master/BUILD @@ -15,6 +15,7 @@ go_library( "apiclient.go", "discovery.go", "manifests.go", + "selfhosted.go", "tokens.go", ], tags = ["automanaged"], diff --git a/cmd/kubeadm/app/master/apiclient.go b/cmd/kubeadm/app/master/apiclient.go index 4f1a67f2088..cdbadd168bc 100644 --- a/cmd/kubeadm/app/master/apiclient.go +++ b/cmd/kubeadm/app/master/apiclient.go @@ -65,38 +65,12 @@ func CreateClientAndWaitForAPI(file string) (*clientset.Clientset, error) { if err != nil { return nil, err } + fmt.Println("[apiclient] Created API client, waiting for the control plane to become ready") - - start := time.Now() - wait.PollInfinite(apiCallRetryInterval, func() (bool, error) { - // TODO: use /healthz API instead of this - cs, err := client.ComponentStatuses().List(v1.ListOptions{}) - if err != nil { - if apierrs.IsForbidden(err) { - fmt.Println("[apiclient] Waiting for API server authorization") - } - return false, nil - } - // TODO(phase2) must revisit this when we implement HA - if len(cs.Items) < 3 { - fmt.Println("[apiclient] Not all control plane components are ready yet") - return false, nil - } - for _, item := range cs.Items { - for _, condition := range item.Conditions { - if condition.Type != v1.ComponentHealthy { - fmt.Printf("[apiclient] Control plane component %q is still unhealthy: %#v\n", item.ObjectMeta.Name, item.Conditions) - return false, nil - } - } - } - - fmt.Printf("[apiclient] All control plane components are healthy after %f seconds\n", time.Since(start).Seconds()) - return true, nil - }) + WaitForAPI(client) fmt.Println("[apiclient] Waiting for at least one node to register and become ready") - start = time.Now() + start := time.Now() wait.PollInfinite(apiCallRetryInterval, func() (bool, error) { nodeList, err := client.Nodes().List(v1.ListOptions{}) if err != nil { @@ -128,6 +102,36 @@ func standardLabels(n string) map[string]string { } } +func WaitForAPI(client *clientset.Clientset) { + start := time.Now() + wait.PollInfinite(apiCallRetryInterval, func() (bool, error) { + // TODO: use /healthz API instead of this + cs, err := client.ComponentStatuses().List(v1.ListOptions{}) + if err != nil { + if apierrs.IsForbidden(err) { + fmt.Print("\r[apiclient] Waiting for the API server to create RBAC policies") + } + return false, nil + } + fmt.Println("\n[apiclient] RBAC policies created") + // TODO(phase2) must revisit this when we implement HA + if len(cs.Items) < 3 { + return false, nil + } + for _, item := range cs.Items { + for _, condition := range item.Conditions { + if condition.Type != v1.ComponentHealthy { + fmt.Printf("[apiclient] Control plane component %q is still unhealthy: %#v\n", item.ObjectMeta.Name, item.Conditions) + return false, nil + } + } + } + + fmt.Printf("[apiclient] All control plane components are healthy after %f seconds\n", time.Since(start).Seconds()) + return true, nil + }) +} + func NewDaemonSet(daemonName string, podSpec v1.PodSpec) *extensions.DaemonSet { l := standardLabels(daemonName) return &extensions.DaemonSet{ diff --git a/cmd/kubeadm/app/master/manifests.go b/cmd/kubeadm/app/master/manifests.go index 069cb0f669c..e5dc2e83c11 100644 --- a/cmd/kubeadm/app/master/manifests.go +++ b/cmd/kubeadm/app/master/manifests.go @@ -98,10 +98,12 @@ func WriteStaticPodManifests(cfg *kubeadmapi.MasterConfiguration) error { Env: getProxyEnvVars(), }, volumes...), kubeScheduler: componentPod(api.Container{ - Name: kubeScheduler, - Image: images.GetCoreImage(images.KubeSchedulerImage, cfg, kubeadmapi.GlobalEnvParams.HyperkubeImage), - Command: getSchedulerCommand(cfg), - LivenessProbe: componentProbe(10251, "/healthz"), + Name: kubeScheduler, + Image: images.GetCoreImage(images.KubeSchedulerImage, cfg, kubeadmapi.GlobalEnvParams.HyperkubeImage), + // TODO: Using non-standard port here so self-hosted scheduler can come up: + // Use the regular port if this is not going to be a self-hosted deployment. + Command: getSchedulerCommand(cfg, 10260), + LivenessProbe: componentProbe(10260, "/healthz"), Resources: componentResources("100m"), Env: getProxyEnvVars(), }), @@ -389,11 +391,12 @@ func getControllerManagerCommand(cfg *kubeadmapi.MasterConfiguration) []string { return command } -func getSchedulerCommand(cfg *kubeadmapi.MasterConfiguration) []string { +func getSchedulerCommand(cfg *kubeadmapi.MasterConfiguration, schedulerPort int) []string { return append(getComponentBaseCommand(scheduler), "--address=127.0.0.1", "--leader-elect", "--master=127.0.0.1:8080", + fmt.Sprintf("--port=%d", schedulerPort), ) } diff --git a/cmd/kubeadm/app/master/selfhosted.go b/cmd/kubeadm/app/master/selfhosted.go new file mode 100644 index 00000000000..6a30aae7baa --- /dev/null +++ b/cmd/kubeadm/app/master/selfhosted.go @@ -0,0 +1,334 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package master + +import ( + "encoding/json" + "fmt" + "os" + "path" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" + "k8s.io/kubernetes/cmd/kubeadm/app/images" + "k8s.io/kubernetes/pkg/api" + "k8s.io/kubernetes/pkg/api/v1" + ext "k8s.io/kubernetes/pkg/apis/extensions/v1beta1" + "k8s.io/kubernetes/pkg/client/clientset_generated/clientset" +) + +func CreateSelfHostedControlPlane(cfg *kubeadmapi.MasterConfiguration, client *clientset.Clientset) error { + volumes := []v1.Volume{k8sVolume(cfg)} + volumeMounts := []v1.VolumeMount{k8sVolumeMount()} + if isCertsVolumeMountNeeded() { + volumes = append(volumes, certsVolume(cfg)) + volumeMounts = append(volumeMounts, certsVolumeMount()) + } + + if isPkiVolumeMountNeeded() { + volumes = append(volumes, pkiVolume(cfg)) + volumeMounts = append(volumeMounts, pkiVolumeMount()) + } + + if err := LaunchSelfHostedAPIServer(cfg, client, volumes, volumeMounts); err != nil { + return err + } + + if err := LaunchSelfHostedScheduler(cfg, client, volumes, volumeMounts); err != nil { + return err + } + + if err := LaunchSelfHostedControllerManager(cfg, client, volumes, volumeMounts); err != nil { + return err + } + return nil +} + +func LaunchSelfHostedAPIServer(cfg *kubeadmapi.MasterConfiguration, client *clientset.Clientset, volumes []v1.Volume, volumeMounts []v1.VolumeMount) error { + start := time.Now() + + apiServer := getAPIServerDS(cfg, volumes, volumeMounts) + if _, err := client.Extensions().DaemonSets(api.NamespaceSystem).Create(&apiServer); err != nil { + return fmt.Errorf("failed to create self-hosted %q daemon set [%v]", kubeAPIServer, err) + } + + wait.PollInfinite(apiCallRetryInterval, func() (bool, error) { + // TODO: This might be pointless, checking the pods is probably enough. + // It does however get us a count of how many there should be which may be useful + // with HA. + apiDS, err := client.DaemonSets(api.NamespaceSystem).Get(kubeAPIServer, + metav1.GetOptions{}) + if err != nil { + fmt.Println("[debug] error getting apiserver DaemonSet:", err) + return false, nil + } + fmt.Printf("[debug] %s DaemonSet current=%d, desired=%d\n", + kubeAPIServer, + apiDS.Status.CurrentNumberScheduled, + apiDS.Status.DesiredNumberScheduled) + + if apiDS.Status.CurrentNumberScheduled != apiDS.Status.DesiredNumberScheduled { + return false, nil + } + + return true, nil + }) + + waitForPodsWithLabel(client, kubeAPIServer) + + apiServerStaticManifestPath := path.Join(kubeadmapi.GlobalEnvParams.KubernetesDir, + "manifests", kubeAPIServer+".json") + if err := os.Remove(apiServerStaticManifestPath); err != nil { + return fmt.Errorf("unable to delete temporary API server manifest [%v]", err) + } + + // Wait until kubernetes detects the static pod removal and our newly created + // API server comes online: + // TODO: Should we verify that either the API is down, or the static apiserver pod is gone before + // waiting? + WaitForAPI(client) + + fmt.Printf("[debug] self-hosted kube-apiserver ready after %f seconds\n", time.Since(start).Seconds()) + return nil +} + +func LaunchSelfHostedControllerManager(cfg *kubeadmapi.MasterConfiguration, client *clientset.Clientset, volumes []v1.Volume, volumeMounts []v1.VolumeMount) error { + start := time.Now() + + ctrlMgr := getControllerManagerDeployment(cfg, volumes, volumeMounts) + if _, err := client.Extensions().Deployments(api.NamespaceSystem).Create(&ctrlMgr); err != nil { + return fmt.Errorf("failed to create self-hosted %q deployment [%v]", kubeControllerManager, err) + } + + waitForPodsWithLabel(client, kubeControllerManager) + + ctrlMgrStaticManifestPath := path.Join(kubeadmapi.GlobalEnvParams.KubernetesDir, + "manifests", kubeControllerManager+".json") + if err := os.Remove(ctrlMgrStaticManifestPath); err != nil { + return fmt.Errorf("unable to delete temporary controller manager manifest [%v]", err) + } + + fmt.Printf("[debug] self-hosted kube-controller-manager ready after %f seconds\n", time.Since(start).Seconds()) + return nil + +} + +func LaunchSelfHostedScheduler(cfg *kubeadmapi.MasterConfiguration, client *clientset.Clientset, volumes []v1.Volume, volumeMounts []v1.VolumeMount) error { + + start := time.Now() + scheduler := getSchedulerDeployment(cfg) + if _, err := client.Extensions().Deployments(api.NamespaceSystem).Create(&scheduler); err != nil { + return fmt.Errorf("failed to create self-hosted %q deployment [%v]", kubeScheduler, err) + } + + waitForPodsWithLabel(client, kubeScheduler) + + schedulerStaticManifestPath := path.Join(kubeadmapi.GlobalEnvParams.KubernetesDir, + "manifests", kubeScheduler+".json") + if err := os.Remove(schedulerStaticManifestPath); err != nil { + return fmt.Errorf("unable to delete temporary scheduler manifest [%v]", err) + } + + fmt.Printf("[debug] self-hosted kube-scheduler ready after %f seconds\n", time.Since(start).Seconds()) + return nil +} + +// waitForPodsWithLabel will lookup pods with the given label and wait until they are all +// reporting status as running. +func waitForPodsWithLabel(client *clientset.Clientset, appLabel string) { + wait.PollInfinite(apiCallRetryInterval, func() (bool, error) { + // TODO: Do we need a stronger label link than this? + listOpts := v1.ListOptions{LabelSelector: fmt.Sprintf("k8s-app=%s", appLabel)} + apiPods, err := client.Pods(api.NamespaceSystem).List(listOpts) + if err != nil { + fmt.Printf("[debug] error getting %s pods [%v]\n", appLabel, err) + return false, nil + } + fmt.Printf("[debug] Found %d %s pods\n", len(apiPods.Items), appLabel) + + // TODO: HA + if int32(len(apiPods.Items)) != 1 { + return false, nil + } + for _, pod := range apiPods.Items { + fmt.Printf("[debug] Pod %s status: %s\n", pod.Name, pod.Status.Phase) + if pod.Status.Phase != "Running" { + return false, nil + } + } + + return true, nil + }) + + return +} + +// Sources from bootkube templates.go +func getAPIServerDS(cfg *kubeadmapi.MasterConfiguration, + volumes []v1.Volume, volumeMounts []v1.VolumeMount) ext.DaemonSet { + + ds := ext.DaemonSet{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "extensions/v1beta1", + Kind: "DaemonSet", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: kubeAPIServer, + Namespace: "kube-system", + //Labels: map[string]string{"k8s-app": "kube-apiserver"}, + }, + Spec: ext.DaemonSetSpec{ + Template: v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + // TODO: taken from bootkube, appears to be essential, without this + // we don't get an apiserver pod... + "k8s-app": kubeAPIServer, + "component": kubeAPIServer, + "tier": "control-plane", + }, + }, + Spec: v1.PodSpec{ + // TODO: Make sure masters get this label + NodeSelector: map[string]string{metav1.NodeLabelKubeadmAlphaRole: metav1.NodeLabelRoleMaster}, + HostNetwork: true, + Volumes: volumes, + Containers: []v1.Container{ + { + Name: kubeAPIServer, + Image: images.GetCoreImage(images.KubeAPIServerImage, cfg, kubeadmapi.GlobalEnvParams.HyperkubeImage), + Command: getAPIServerCommand(cfg), + Env: getProxyEnvVars(), + VolumeMounts: volumeMounts, + LivenessProbe: componentProbe(8080, "/healthz"), + Resources: componentResources("250m"), + }, + }, + }, + }, + }, + } + return ds +} + +func getControllerManagerDeployment(cfg *kubeadmapi.MasterConfiguration, + volumes []v1.Volume, volumeMounts []v1.VolumeMount) ext.Deployment { + + cmDep := ext.Deployment{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "extensions/v1beta1", + Kind: "Deployment", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: kubeControllerManager, + Namespace: "kube-system", + }, + Spec: ext.DeploymentSpec{ + Template: v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + // TODO: taken from bootkube, appears to be essential + "k8s-app": kubeControllerManager, + "component": kubeControllerManager, + "tier": "control-plane", + }, + Annotations: map[string]string{ + v1.TolerationsAnnotationKey: getMasterToleration(), + }, + }, + Spec: v1.PodSpec{ + // TODO: Make sure masters get this label + NodeSelector: map[string]string{metav1.NodeLabelKubeadmAlphaRole: metav1.NodeLabelRoleMaster}, + HostNetwork: true, + Volumes: volumes, + + Containers: []v1.Container{ + { + Name: kubeControllerManager, + Image: images.GetCoreImage(images.KubeControllerManagerImage, cfg, kubeadmapi.GlobalEnvParams.HyperkubeImage), + Command: getControllerManagerCommand(cfg), + VolumeMounts: volumeMounts, + LivenessProbe: componentProbe(10252, "/healthz"), + Resources: componentResources("200m"), + Env: getProxyEnvVars(), + }, + }, + }, + }, + }, + } + return cmDep +} + +func getMasterToleration() string { + // Tolerate the master taint we add to our master nodes, as this can and should + // run there. + // TODO: Duplicated above + masterToleration, _ := json.Marshal([]v1.Toleration{{ + Key: "dedicated", + Value: "master", + Operator: v1.TolerationOpEqual, + Effect: v1.TaintEffectNoSchedule, + }}) + return string(masterToleration) +} + +func getSchedulerDeployment(cfg *kubeadmapi.MasterConfiguration) ext.Deployment { + + cmDep := ext.Deployment{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "extensions/v1beta1", + Kind: "Deployment", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: kubeScheduler, + Namespace: "kube-system", + }, + Spec: ext.DeploymentSpec{ + Template: v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "k8s-app": kubeScheduler, + "component": kubeScheduler, + "tier": "control-plane", + }, + Annotations: map[string]string{ + v1.TolerationsAnnotationKey: getMasterToleration(), + }, + }, + Spec: v1.PodSpec{ + NodeSelector: map[string]string{metav1.NodeLabelKubeadmAlphaRole: metav1.NodeLabelRoleMaster}, + HostNetwork: true, + + Containers: []v1.Container{ + { + Name: kubeScheduler, + Image: images.GetCoreImage(images.KubeSchedulerImage, cfg, kubeadmapi.GlobalEnvParams.HyperkubeImage), + Command: getSchedulerCommand(cfg, 10251), + LivenessProbe: componentProbe(10251, "/healthz"), + Resources: componentResources("100m"), + Env: getProxyEnvVars(), + }, + }, + }, + }, + }, + } + return cmDep +} From c80c0275da664604d508d2f0949358607475ba61 Mon Sep 17 00:00:00 2001 From: Paulo Pires Date: Fri, 20 Jan 2017 16:26:48 +0000 Subject: [PATCH 2/3] kubeadm: add self-hosted as optional deployment type. --- cmd/kubeadm/app/cmd/init.go | 12 -- cmd/kubeadm/app/master/apiclient.go | 4 +- cmd/kubeadm/app/master/manifests.go | 87 ++++++++-- cmd/kubeadm/app/master/manifests_test.go | 6 +- cmd/kubeadm/app/master/selfhosted.go | 205 +++++++++++------------ 5 files changed, 177 insertions(+), 137 deletions(-) diff --git a/cmd/kubeadm/app/cmd/init.go b/cmd/kubeadm/app/cmd/init.go index cdb7a391aaf..8a25cef156b 100644 --- a/cmd/kubeadm/app/cmd/init.go +++ b/cmd/kubeadm/app/cmd/init.go @@ -173,18 +173,6 @@ func NewInit(cfgPath string, cfg *kubeadmapi.MasterConfiguration, skipPreFlight // Try to start the kubelet service in case it's inactive preflight.TryStartKubelet() - // validate version argument - ver, err := kubeadmutil.KubernetesReleaseVersion(cfg.KubernetesVersion) - if err != nil { - if cfg.KubernetesVersion != kubeadmapiext.DefaultKubernetesVersion { - return nil, err - } else { - ver = kubeadmapiext.DefaultKubernetesFallbackVersion - } - } - cfg.KubernetesVersion = ver - fmt.Println("[init] Using Kubernetes version:", ver) - fmt.Println("[init] Using Authorization mode:", cfg.AuthorizationMode) // Warn about the limitations with the current cloudprovider solution. if cfg.CloudProvider != "" { diff --git a/cmd/kubeadm/app/master/apiclient.go b/cmd/kubeadm/app/master/apiclient.go index cdbadd168bc..b72f948d85c 100644 --- a/cmd/kubeadm/app/master/apiclient.go +++ b/cmd/kubeadm/app/master/apiclient.go @@ -109,11 +109,11 @@ func WaitForAPI(client *clientset.Clientset) { cs, err := client.ComponentStatuses().List(v1.ListOptions{}) if err != nil { if apierrs.IsForbidden(err) { - fmt.Print("\r[apiclient] Waiting for the API server to create RBAC policies") + fmt.Println("[apiclient] Waiting for API server authorization") } return false, nil } - fmt.Println("\n[apiclient] RBAC policies created") + // TODO(phase2) must revisit this when we implement HA if len(cs.Items) < 3 { return false, nil diff --git a/cmd/kubeadm/app/master/manifests.go b/cmd/kubeadm/app/master/manifests.go index e5dc2e83c11..65e9dbd1409 100644 --- a/cmd/kubeadm/app/master/manifests.go +++ b/cmd/kubeadm/app/master/manifests.go @@ -82,7 +82,7 @@ func WriteStaticPodManifests(cfg *kubeadmapi.MasterConfiguration) error { kubeAPIServer: componentPod(api.Container{ Name: kubeAPIServer, Image: images.GetCoreImage(images.KubeAPIServerImage, cfg, kubeadmapi.GlobalEnvParams.HyperkubeImage), - Command: getAPIServerCommand(cfg), + Command: getAPIServerCommand(cfg, false), VolumeMounts: volumeMounts, LivenessProbe: componentProbe(8080, "/healthz"), Resources: componentResources("250m"), @@ -91,19 +91,17 @@ func WriteStaticPodManifests(cfg *kubeadmapi.MasterConfiguration) error { kubeControllerManager: componentPod(api.Container{ Name: kubeControllerManager, Image: images.GetCoreImage(images.KubeControllerManagerImage, cfg, kubeadmapi.GlobalEnvParams.HyperkubeImage), - Command: getControllerManagerCommand(cfg), + Command: getControllerManagerCommand(cfg, false), VolumeMounts: volumeMounts, LivenessProbe: componentProbe(10252, "/healthz"), Resources: componentResources("200m"), Env: getProxyEnvVars(), }, volumes...), kubeScheduler: componentPod(api.Container{ - Name: kubeScheduler, - Image: images.GetCoreImage(images.KubeSchedulerImage, cfg, kubeadmapi.GlobalEnvParams.HyperkubeImage), - // TODO: Using non-standard port here so self-hosted scheduler can come up: - // Use the regular port if this is not going to be a self-hosted deployment. - Command: getSchedulerCommand(cfg, 10260), - LivenessProbe: componentProbe(10260, "/healthz"), + Name: kubeScheduler, + Image: images.GetCoreImage(images.KubeSchedulerImage, cfg, kubeadmapi.GlobalEnvParams.HyperkubeImage), + Command: getSchedulerCommand(cfg, false), + LivenessProbe: componentProbe(10251, "/healthz"), Resources: componentResources("100m"), Env: getProxyEnvVars(), }), @@ -219,6 +217,23 @@ func pkiVolumeMount() api.VolumeMount { } } +func flockVolume() api.Volume { + return api.Volume{ + Name: "var-lock", + VolumeSource: api.VolumeSource{ + HostPath: &api.HostPathVolumeSource{Path: "/var/lock"}, + }, + } +} + +func flockVolumeMount() api.VolumeMount { + return api.VolumeMount{ + Name: "var-lock", + MountPath: "/var/lock", + ReadOnly: false, + } +} + func k8sVolume(cfg *kubeadmapi.MasterConfiguration) api.Volume { return api.Volume{ Name: "k8s", @@ -286,8 +301,15 @@ func getComponentBaseCommand(component string) []string { return []string{"kube-" + component} } -func getAPIServerCommand(cfg *kubeadmapi.MasterConfiguration) []string { - command := append(getComponentBaseCommand(apiServer), +func getAPIServerCommand(cfg *kubeadmapi.MasterConfiguration, selfHosted bool) []string { + var command []string + + // self-hosted apiserver needs to wait on a lock + if selfHosted { + command = []string{"/usr/bin/flock", "--exclusive", "--timeout=30", "/var/lock/api-server.lock"} + } + + command = append(getComponentBaseCommand(apiServer), "--insecure-bind-address=127.0.0.1", "--admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota", "--service-cluster-ip-range="+cfg.Networking.ServiceSubnet, @@ -312,7 +334,11 @@ func getAPIServerCommand(cfg *kubeadmapi.MasterConfiguration) []string { // Use first address we are given if len(cfg.API.AdvertiseAddresses) > 0 { - command = append(command, fmt.Sprintf("--advertise-address=%s", cfg.API.AdvertiseAddresses[0])) + if selfHosted { + command = append(command, "--advertise-address=$(POD_IP)") + } else { + command = append(command, fmt.Sprintf("--advertise-address=%s", cfg.API.AdvertiseAddresses[0])) + } } if len(cfg.KubernetesVersion) != 0 { @@ -361,8 +387,15 @@ func getAPIServerCommand(cfg *kubeadmapi.MasterConfiguration) []string { return command } -func getControllerManagerCommand(cfg *kubeadmapi.MasterConfiguration) []string { - command := append(getComponentBaseCommand(controllerManager), +func getControllerManagerCommand(cfg *kubeadmapi.MasterConfiguration, selfHosted bool) []string { + var command []string + + // self-hosted controller-manager needs to wait on a lock + if selfHosted { + command = []string{"/usr/bin/flock", "--exclusive", "--timeout=30", "/var/lock/controller-manager.lock"} + } + + command = append(getComponentBaseCommand(controllerManager), "--address=127.0.0.1", "--leader-elect", "--master=127.0.0.1:8080", @@ -388,16 +421,25 @@ func getControllerManagerCommand(cfg *kubeadmapi.MasterConfiguration) []string { if cfg.Networking.PodSubnet != "" { command = append(command, "--allocate-node-cidrs=true", "--cluster-cidr="+cfg.Networking.PodSubnet) } + return command } -func getSchedulerCommand(cfg *kubeadmapi.MasterConfiguration, schedulerPort int) []string { - return append(getComponentBaseCommand(scheduler), +func getSchedulerCommand(cfg *kubeadmapi.MasterConfiguration, selfHosted bool) []string { + var command []string + + // self-hosted apiserver needs to wait on a lock + if selfHosted { + command = []string{"/usr/bin/flock", "--exclusive", "--timeout=30", "/var/lock/api-server.lock"} + } + + command = append(getComponentBaseCommand(scheduler), "--address=127.0.0.1", "--leader-elect", "--master=127.0.0.1:8080", - fmt.Sprintf("--port=%d", schedulerPort), ) + + return command } func getProxyCommand(cfg *kubeadmapi.MasterConfiguration) []string { @@ -421,3 +463,16 @@ func getProxyEnvVars() []api.EnvVar { } return envs } + +func getSelfHostedAPIServerEnv() []api.EnvVar { + podIPEnvVar := api.EnvVar{ + Name: "POD_IP", + ValueFrom: &api.EnvVarSource{ + FieldRef: &api.ObjectFieldSelector{ + FieldPath: "status.podIP", + }, + }, + } + + return append(getProxyEnvVars(), podIPEnvVar) +} diff --git a/cmd/kubeadm/app/master/manifests_test.go b/cmd/kubeadm/app/master/manifests_test.go index 214bf0424c3..11d028d6678 100644 --- a/cmd/kubeadm/app/master/manifests_test.go +++ b/cmd/kubeadm/app/master/manifests_test.go @@ -454,7 +454,7 @@ func TestGetAPIServerCommand(t *testing.T) { } for _, rt := range tests { - actual := getAPIServerCommand(rt.cfg) + actual := getAPIServerCommand(rt.cfg, false) for i := range actual { if actual[i] != rt.expected[i] { t.Errorf( @@ -523,7 +523,7 @@ func TestGetControllerManagerCommand(t *testing.T) { } for _, rt := range tests { - actual := getControllerManagerCommand(rt.cfg) + actual := getControllerManagerCommand(rt.cfg, false) for i := range actual { if actual[i] != rt.expected[i] { t.Errorf( @@ -553,7 +553,7 @@ func TestGetSchedulerCommand(t *testing.T) { } for _, rt := range tests { - actual := getSchedulerCommand(rt.cfg) + actual := getSchedulerCommand(rt.cfg, false) for i := range actual { if actual[i] != rt.expected[i] { t.Errorf( diff --git a/cmd/kubeadm/app/master/selfhosted.go b/cmd/kubeadm/app/master/selfhosted.go index 6a30aae7baa..b0c54e48c36 100644 --- a/cmd/kubeadm/app/master/selfhosted.go +++ b/cmd/kubeadm/app/master/selfhosted.go @@ -46,21 +46,26 @@ func CreateSelfHostedControlPlane(cfg *kubeadmapi.MasterConfiguration, client *c volumeMounts = append(volumeMounts, pkiVolumeMount()) } - if err := LaunchSelfHostedAPIServer(cfg, client, volumes, volumeMounts); err != nil { + // Need lock for self-hosted + volumes = append(volumes, flockVolume()) + volumeMounts = append(volumeMounts, flockVolumeMount()) + + if err := launchSelfHostedAPIServer(cfg, client, volumes, volumeMounts); err != nil { return err } - if err := LaunchSelfHostedScheduler(cfg, client, volumes, volumeMounts); err != nil { + if err := launchSelfHostedScheduler(cfg, client, volumes, volumeMounts); err != nil { return err } - if err := LaunchSelfHostedControllerManager(cfg, client, volumes, volumeMounts); err != nil { + if err := launchSelfHostedControllerManager(cfg, client, volumes, volumeMounts); err != nil { return err } + return nil } -func LaunchSelfHostedAPIServer(cfg *kubeadmapi.MasterConfiguration, client *clientset.Clientset, volumes []v1.Volume, volumeMounts []v1.VolumeMount) error { +func launchSelfHostedAPIServer(cfg *kubeadmapi.MasterConfiguration, client *clientset.Clientset, volumes []v1.Volume, volumeMounts []v1.VolumeMount) error { start := time.Now() apiServer := getAPIServerDS(cfg, volumes, volumeMounts) @@ -72,13 +77,13 @@ func LaunchSelfHostedAPIServer(cfg *kubeadmapi.MasterConfiguration, client *clie // TODO: This might be pointless, checking the pods is probably enough. // It does however get us a count of how many there should be which may be useful // with HA. - apiDS, err := client.DaemonSets(api.NamespaceSystem).Get(kubeAPIServer, + apiDS, err := client.DaemonSets(api.NamespaceSystem).Get("self-hosted-"+kubeAPIServer, metav1.GetOptions{}) if err != nil { - fmt.Println("[debug] error getting apiserver DaemonSet:", err) + fmt.Println("[self-hosted] error getting apiserver DaemonSet:", err) return false, nil } - fmt.Printf("[debug] %s DaemonSet current=%d, desired=%d\n", + fmt.Printf("[self-hosted] %s DaemonSet current=%d, desired=%d\n", kubeAPIServer, apiDS.Status.CurrentNumberScheduled, apiDS.Status.DesiredNumberScheduled) @@ -90,25 +95,22 @@ func LaunchSelfHostedAPIServer(cfg *kubeadmapi.MasterConfiguration, client *clie return true, nil }) - waitForPodsWithLabel(client, kubeAPIServer) + // Wait for self-hosted API server to take ownership + waitForPodsWithLabel(client, "self-hosted-"+kubeAPIServer, true) - apiServerStaticManifestPath := path.Join(kubeadmapi.GlobalEnvParams.KubernetesDir, - "manifests", kubeAPIServer+".json") - if err := os.Remove(apiServerStaticManifestPath); err != nil { + // Remove temporary API server + apiServerStaticManifestPath := buildStaticManifestFilepath(kubeAPIServer) + if err := os.RemoveAll(apiServerStaticManifestPath); err != nil { return fmt.Errorf("unable to delete temporary API server manifest [%v]", err) } - // Wait until kubernetes detects the static pod removal and our newly created - // API server comes online: - // TODO: Should we verify that either the API is down, or the static apiserver pod is gone before - // waiting? WaitForAPI(client) - fmt.Printf("[debug] self-hosted kube-apiserver ready after %f seconds\n", time.Since(start).Seconds()) + fmt.Printf("[self-hosted] self-hosted kube-apiserver ready after %f seconds\n", time.Since(start).Seconds()) return nil } -func LaunchSelfHostedControllerManager(cfg *kubeadmapi.MasterConfiguration, client *clientset.Clientset, volumes []v1.Volume, volumeMounts []v1.VolumeMount) error { +func launchSelfHostedControllerManager(cfg *kubeadmapi.MasterConfiguration, client *clientset.Clientset, volumes []v1.Volume, volumeMounts []v1.VolumeMount) error { start := time.Now() ctrlMgr := getControllerManagerDeployment(cfg, volumes, volumeMounts) @@ -116,105 +118,98 @@ func LaunchSelfHostedControllerManager(cfg *kubeadmapi.MasterConfiguration, clie return fmt.Errorf("failed to create self-hosted %q deployment [%v]", kubeControllerManager, err) } - waitForPodsWithLabel(client, kubeControllerManager) + waitForPodsWithLabel(client, "self-hosted-"+kubeControllerManager, false) - ctrlMgrStaticManifestPath := path.Join(kubeadmapi.GlobalEnvParams.KubernetesDir, - "manifests", kubeControllerManager+".json") - if err := os.Remove(ctrlMgrStaticManifestPath); err != nil { + ctrlMgrStaticManifestPath := buildStaticManifestFilepath(kubeControllerManager) + if err := os.RemoveAll(ctrlMgrStaticManifestPath); err != nil { return fmt.Errorf("unable to delete temporary controller manager manifest [%v]", err) } - fmt.Printf("[debug] self-hosted kube-controller-manager ready after %f seconds\n", time.Since(start).Seconds()) + fmt.Printf("[self-hosted] self-hosted kube-controller-manager ready after %f seconds\n", time.Since(start).Seconds()) return nil } -func LaunchSelfHostedScheduler(cfg *kubeadmapi.MasterConfiguration, client *clientset.Clientset, volumes []v1.Volume, volumeMounts []v1.VolumeMount) error { - +func launchSelfHostedScheduler(cfg *kubeadmapi.MasterConfiguration, client *clientset.Clientset, volumes []v1.Volume, volumeMounts []v1.VolumeMount) error { start := time.Now() scheduler := getSchedulerDeployment(cfg) if _, err := client.Extensions().Deployments(api.NamespaceSystem).Create(&scheduler); err != nil { return fmt.Errorf("failed to create self-hosted %q deployment [%v]", kubeScheduler, err) } - waitForPodsWithLabel(client, kubeScheduler) + waitForPodsWithLabel(client, "self-hosted-"+kubeScheduler, false) - schedulerStaticManifestPath := path.Join(kubeadmapi.GlobalEnvParams.KubernetesDir, - "manifests", kubeScheduler+".json") - if err := os.Remove(schedulerStaticManifestPath); err != nil { + schedulerStaticManifestPath := buildStaticManifestFilepath(kubeScheduler) + if err := os.RemoveAll(schedulerStaticManifestPath); err != nil { return fmt.Errorf("unable to delete temporary scheduler manifest [%v]", err) } - fmt.Printf("[debug] self-hosted kube-scheduler ready after %f seconds\n", time.Since(start).Seconds()) + fmt.Printf("[self-hosted] self-hosted kube-scheduler ready after %f seconds\n", time.Since(start).Seconds()) return nil } // waitForPodsWithLabel will lookup pods with the given label and wait until they are all // reporting status as running. -func waitForPodsWithLabel(client *clientset.Clientset, appLabel string) { +func waitForPodsWithLabel(client *clientset.Clientset, appLabel string, mustBeRunning bool) { wait.PollInfinite(apiCallRetryInterval, func() (bool, error) { // TODO: Do we need a stronger label link than this? listOpts := v1.ListOptions{LabelSelector: fmt.Sprintf("k8s-app=%s", appLabel)} apiPods, err := client.Pods(api.NamespaceSystem).List(listOpts) if err != nil { - fmt.Printf("[debug] error getting %s pods [%v]\n", appLabel, err) + fmt.Printf("[self-hosted] error getting %s pods [%v]\n", appLabel, err) return false, nil } - fmt.Printf("[debug] Found %d %s pods\n", len(apiPods.Items), appLabel) + fmt.Printf("[self-hosted] Found %d %s pods\n", len(apiPods.Items), appLabel) // TODO: HA if int32(len(apiPods.Items)) != 1 { return false, nil } for _, pod := range apiPods.Items { - fmt.Printf("[debug] Pod %s status: %s\n", pod.Name, pod.Status.Phase) - if pod.Status.Phase != "Running" { + fmt.Printf("[self-hosted] Pod %s status: %s\n", pod.Name, pod.Status.Phase) + if mustBeRunning && pod.Status.Phase != "Running" { return false, nil } } return true, nil }) - - return } // Sources from bootkube templates.go -func getAPIServerDS(cfg *kubeadmapi.MasterConfiguration, - volumes []v1.Volume, volumeMounts []v1.VolumeMount) ext.DaemonSet { - +func getAPIServerDS(cfg *kubeadmapi.MasterConfiguration, volumes []v1.Volume, volumeMounts []v1.VolumeMount) ext.DaemonSet { ds := ext.DaemonSet{ TypeMeta: metav1.TypeMeta{ APIVersion: "extensions/v1beta1", Kind: "DaemonSet", }, ObjectMeta: metav1.ObjectMeta{ - Name: kubeAPIServer, + Name: "self-hosted-" + kubeAPIServer, Namespace: "kube-system", - //Labels: map[string]string{"k8s-app": "kube-apiserver"}, + Labels: map[string]string{"k8s-app": "self-hosted-" + kubeAPIServer}, }, Spec: ext.DaemonSetSpec{ Template: v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{ - // TODO: taken from bootkube, appears to be essential, without this - // we don't get an apiserver pod... - "k8s-app": kubeAPIServer, + "k8s-app": "self-hosted-" + kubeAPIServer, "component": kubeAPIServer, "tier": "control-plane", }, + Annotations: map[string]string{ + v1.TolerationsAnnotationKey: getMasterToleration(), + }, }, Spec: v1.PodSpec{ - // TODO: Make sure masters get this label NodeSelector: map[string]string{metav1.NodeLabelKubeadmAlphaRole: metav1.NodeLabelRoleMaster}, HostNetwork: true, Volumes: volumes, Containers: []v1.Container{ { - Name: kubeAPIServer, + Name: "self-hosted-" + kubeAPIServer, Image: images.GetCoreImage(images.KubeAPIServerImage, cfg, kubeadmapi.GlobalEnvParams.HyperkubeImage), - Command: getAPIServerCommand(cfg), - Env: getProxyEnvVars(), + Command: getAPIServerCommand(cfg, true), + Env: getSelfHostedAPIServerEnv(), VolumeMounts: volumeMounts, LivenessProbe: componentProbe(8080, "/healthz"), Resources: componentResources("250m"), @@ -227,24 +222,23 @@ func getAPIServerDS(cfg *kubeadmapi.MasterConfiguration, return ds } -func getControllerManagerDeployment(cfg *kubeadmapi.MasterConfiguration, - volumes []v1.Volume, volumeMounts []v1.VolumeMount) ext.Deployment { - - cmDep := ext.Deployment{ +func getControllerManagerDeployment(cfg *kubeadmapi.MasterConfiguration, volumes []v1.Volume, volumeMounts []v1.VolumeMount) ext.Deployment { + d := ext.Deployment{ TypeMeta: metav1.TypeMeta{ APIVersion: "extensions/v1beta1", Kind: "Deployment", }, ObjectMeta: metav1.ObjectMeta{ - Name: kubeControllerManager, + Name: "self-hosted-" + kubeControllerManager, Namespace: "kube-system", + Labels: map[string]string{"k8s-app": "self-hosted-" + kubeControllerManager}, }, Spec: ext.DeploymentSpec{ + // TODO bootkube uses 2 replicas Template: v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{ - // TODO: taken from bootkube, appears to be essential - "k8s-app": kubeControllerManager, + "k8s-app": "self-hosted-" + kubeControllerManager, "component": kubeControllerManager, "tier": "control-plane", }, @@ -253,27 +247,74 @@ func getControllerManagerDeployment(cfg *kubeadmapi.MasterConfiguration, }, }, Spec: v1.PodSpec{ - // TODO: Make sure masters get this label NodeSelector: map[string]string{metav1.NodeLabelKubeadmAlphaRole: metav1.NodeLabelRoleMaster}, HostNetwork: true, Volumes: volumes, - Containers: []v1.Container{ { - Name: kubeControllerManager, + Name: "self-hosted-" + kubeControllerManager, Image: images.GetCoreImage(images.KubeControllerManagerImage, cfg, kubeadmapi.GlobalEnvParams.HyperkubeImage), - Command: getControllerManagerCommand(cfg), + Command: getControllerManagerCommand(cfg, true), VolumeMounts: volumeMounts, LivenessProbe: componentProbe(10252, "/healthz"), Resources: componentResources("200m"), Env: getProxyEnvVars(), }, }, + DNSPolicy: v1.DNSDefault, + }, + }, + }, + } + return d +} + +func getSchedulerDeployment(cfg *kubeadmapi.MasterConfiguration) ext.Deployment { + d := ext.Deployment{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "extensions/v1beta1", + Kind: "Deployment", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "self-hosted-" + kubeScheduler, + Namespace: "kube-system", + Labels: map[string]string{"k8s-app": "self-hosted-" + kubeScheduler}, + }, + Spec: ext.DeploymentSpec{ + // TODO bootkube uses 2 replicas + Template: v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "k8s-app": "self-hosted-" + kubeScheduler, + "component": kubeScheduler, + "tier": "control-plane", + }, + Annotations: map[string]string{ + v1.TolerationsAnnotationKey: getMasterToleration(), + }, + }, + Spec: v1.PodSpec{ + NodeSelector: map[string]string{metav1.NodeLabelKubeadmAlphaRole: metav1.NodeLabelRoleMaster}, + HostNetwork: true, + Containers: []v1.Container{ + { + Name: "self-hosted-" + kubeScheduler, + Image: images.GetCoreImage(images.KubeSchedulerImage, cfg, kubeadmapi.GlobalEnvParams.HyperkubeImage), + Command: getSchedulerCommand(cfg, true), + LivenessProbe: componentProbe(10251, "/healthz"), + Resources: componentResources("100m"), + Env: getProxyEnvVars(), + }, + }, }, }, }, } - return cmDep + return d +} + +func buildStaticManifestFilepath(name string) string { + return path.Join(kubeadmapi.GlobalEnvParams.KubernetesDir, "manifests", name+".json") } func getMasterToleration() string { @@ -288,47 +329,3 @@ func getMasterToleration() string { }}) return string(masterToleration) } - -func getSchedulerDeployment(cfg *kubeadmapi.MasterConfiguration) ext.Deployment { - - cmDep := ext.Deployment{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "extensions/v1beta1", - Kind: "Deployment", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: kubeScheduler, - Namespace: "kube-system", - }, - Spec: ext.DeploymentSpec{ - Template: v1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "k8s-app": kubeScheduler, - "component": kubeScheduler, - "tier": "control-plane", - }, - Annotations: map[string]string{ - v1.TolerationsAnnotationKey: getMasterToleration(), - }, - }, - Spec: v1.PodSpec{ - NodeSelector: map[string]string{metav1.NodeLabelKubeadmAlphaRole: metav1.NodeLabelRoleMaster}, - HostNetwork: true, - - Containers: []v1.Container{ - { - Name: kubeScheduler, - Image: images.GetCoreImage(images.KubeSchedulerImage, cfg, kubeadmapi.GlobalEnvParams.HyperkubeImage), - Command: getSchedulerCommand(cfg, 10251), - LivenessProbe: componentProbe(10251, "/healthz"), - Resources: componentResources("100m"), - Env: getProxyEnvVars(), - }, - }, - }, - }, - }, - } - return cmDep -} From 724ce6a8a50f4b62f3bc9b833de67fea2456234c Mon Sep 17 00:00:00 2001 From: Paulo Pires Date: Fri, 20 Jan 2017 16:27:34 +0000 Subject: [PATCH 3/3] kubeadm: add temporary --self-hosted flag. --- cmd/kubeadm/app/cmd/init.go | 159 +++++------------------------- hack/verify-flags/known-flags.txt | 1 + 2 files changed, 23 insertions(+), 137 deletions(-) diff --git a/cmd/kubeadm/app/cmd/init.go b/cmd/kubeadm/app/cmd/init.go index 8a25cef156b..f08c47b59a2 100644 --- a/cmd/kubeadm/app/cmd/init.go +++ b/cmd/kubeadm/app/cmd/init.go @@ -21,13 +21,11 @@ import ( "io" "io/ioutil" "path" - "strconv" "github.com/renstrom/dedent" "github.com/spf13/cobra" "k8s.io/apimachinery/pkg/runtime" - netutil "k8s.io/apimachinery/pkg/util/net" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmapiext "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha1" "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation" @@ -54,9 +52,6 @@ var ( kubeadm join --discovery %s `) - deploymentStaticPod = "static-pods" - deploymentSelfHosted = "self-hosted" - deploymentTypes = []string{deploymentStaticPod, deploymentSelfHosted} ) // NewCmdInit returns "kubeadm init" command. @@ -68,14 +63,14 @@ func NewCmdInit(out io.Writer) *cobra.Command { var cfgPath string var skipPreFlight bool - var deploymentType string // static pods, self-hosted, etc. + var selfHosted bool cmd := &cobra.Command{ Use: "init", Short: "Run this in order to set up the Kubernetes master", Run: func(cmd *cobra.Command, args []string) { - i, err := NewInit(cfgPath, &cfg, skipPreFlight, deploymentType) + i, err := NewInit(cfgPath, &cfg, skipPreFlight, selfHosted) kubeadmutil.CheckErr(err) - kubeadmutil.CheckErr(Validate(i.Cfg())) + kubeadmutil.CheckErr(i.Validate()) kubeadmutil.CheckErr(i.Run(out)) }, } @@ -118,7 +113,7 @@ func NewCmdInit(out io.Writer) *cobra.Command { cmd.PersistentFlags().BoolVar( &skipPreFlight, "skip-preflight-checks", skipPreFlight, - "skip preflight checks normally run before modifying the system", + "Skip preflight checks normally run before modifying the system", ) cmd.PersistentFlags().Var( @@ -126,15 +121,15 @@ func NewCmdInit(out io.Writer) *cobra.Command { "The discovery method kubeadm will use for connecting nodes to the master", ) - cmd.PersistentFlags().StringVar( - &deploymentType, "deployment", deploymentType, - fmt.Sprintf("specify a deployment type from %v", deploymentTypes), + cmd.PersistentFlags().BoolVar( + &selfHosted, "self-hosted", selfHosted, + "Enable self-hosted control plane", ) return cmd } -func NewInit(cfgPath string, cfg *kubeadmapi.MasterConfiguration, skipPreFlight bool, deploymentType string) (Init, error) { +func NewInit(cfgPath string, cfg *kubeadmapi.MasterConfiguration, skipPreFlight bool, selfHosted bool) (*Init, error) { fmt.Println("[kubeadm] WARNING: kubeadm is in alpha, please do not use it for production clusters.") @@ -173,52 +168,27 @@ func NewInit(cfgPath string, cfg *kubeadmapi.MasterConfiguration, skipPreFlight // Try to start the kubelet service in case it's inactive preflight.TryStartKubelet() - // Warn about the limitations with the current cloudprovider solution. if cfg.CloudProvider != "" { fmt.Println("WARNING: For cloudprovider integrations to work --cloud-provider must be set for all kubelets in the cluster.") fmt.Println("\t(/etc/systemd/system/kubelet.service.d/10-kubeadm.conf should be edited for this purpose)") } - var deploymentTypeValid bool - for _, supportedDT := range deploymentTypes { - if deploymentType == supportedDT { - deploymentTypeValid = true - } - } - if !deploymentTypeValid { - return nil, fmt.Errorf("%s is not a valid deployment type, you can use any of %v or leave unset to accept the default", deploymentType, deploymentTypes) - } - if deploymentType == deploymentSelfHosted { - fmt.Println("[init] Creating self-hosted Kubernetes deployment...") - return &SelfHostedInit{cfg: cfg}, nil - } - - fmt.Println("[init] Creating static pod Kubernetes deployment...") - return &StaticPodInit{cfg: cfg}, nil + return &Init{cfg: cfg, selfHosted: selfHosted}, nil } -func Validate(cfg *kubeadmapi.MasterConfiguration) error { - return validation.ValidateMasterConfiguration(cfg).ToAggregate() +type Init struct { + cfg *kubeadmapi.MasterConfiguration + selfHosted bool } -// Init structs define implementations of the cluster setup for each supported -// delpoyment type. -type Init interface { - Cfg() *kubeadmapi.MasterConfiguration - Run(out io.Writer) error -} - -type StaticPodInit struct { - cfg *kubeadmapi.MasterConfiguration -} - -func (spi *StaticPodInit) Cfg() *kubeadmapi.MasterConfiguration { - return spi.cfg +// Validate validates configuration passed to "kubeadm init" +func (i *Init) Validate() error { + return validation.ValidateMasterConfiguration(i.cfg).ToAggregate() } // Run executes master node provisioning, including certificates, needed static pod manifests, etc. -func (i *StaticPodInit) Run(out io.Writer) error { +func (i *Init) Run(out io.Writer) error { // PHASE 1: Generate certificates caCert, err := certphase.CreatePKIAssets(i.cfg, kubeadmapi.GlobalEnvParams.HostPKIPath) @@ -287,97 +257,12 @@ func (i *StaticPodInit) Run(out io.Writer) error { } } - if err := kubemaster.CreateEssentialAddons(i.cfg, client); err != nil { - return err - } - - fmt.Fprintf(out, initDoneMsgf, generateJoinArgs(i.cfg)) - return nil -} - -// SelfHostedInit initializes a self-hosted cluster. -type SelfHostedInit struct { - cfg *kubeadmapi.MasterConfiguration -} - -func (spi *SelfHostedInit) Cfg() *kubeadmapi.MasterConfiguration { - return spi.cfg -} - -// Run executes master node provisioning, including certificates, needed pod manifests, etc. -func (i *SelfHostedInit) Run(out io.Writer) error { - // Validate token if any, otherwise generate - if i.cfg.Discovery.Token != nil { - if i.cfg.Discovery.Token.ID != "" && i.cfg.Discovery.Token.Secret != "" { - fmt.Printf("[token-discovery] A token has been provided, validating [%s]\n", kubeadmutil.BearerToken(i.cfg.Discovery.Token)) - if valid, err := kubeadmutil.ValidateToken(i.cfg.Discovery.Token); valid == false { - return err - } - } else { - fmt.Println("[token-discovery] A token has not been provided, generating one") - if err := kubeadmutil.GenerateToken(i.cfg.Discovery.Token); err != nil { - return err - } - } - - // Make sure there is at least one address - if len(i.cfg.Discovery.Token.Addresses) == 0 { - ip, err := netutil.ChooseHostInterface() - if err != nil { - return err - } - i.cfg.Discovery.Token.Addresses = []string{ip.String() + ":" + strconv.Itoa(kubeadmapiext.DefaultDiscoveryBindPort)} - } - - if err := kubemaster.CreateTokenAuthFile(kubeadmutil.BearerToken(i.cfg.Discovery.Token)); err != nil { - return err - } - } - - // PHASE 1: Generate certificates - caCert, err := certphase.CreatePKIAssets(i.cfg, kubeadmapi.GlobalEnvParams.HostPKIPath) - if err != nil { - return err - } - - // PHASE 2: Generate kubeconfig files for the admin and the kubelet - - // TODO this is not great, but there is only one address we can use here - // so we'll pick the first one, there is much of chance to have an empty - // slice by the time this gets called - masterEndpoint := fmt.Sprintf("https://%s:%d", i.cfg.API.AdvertiseAddresses[0], i.cfg.API.Port) - err = kubeconfigphase.CreateAdminAndKubeletKubeConfig(masterEndpoint, kubeadmapi.GlobalEnvParams.HostPKIPath, kubeadmapi.GlobalEnvParams.KubernetesDir) - if err != nil { - return err - } - - // Phase 3: Bootstrap the control plane - if err := kubemaster.WriteStaticPodManifests(i.cfg); err != nil { - return err - } - - client, err := kubemaster.CreateClientAndWaitForAPI(path.Join(kubeadmapi.GlobalEnvParams.KubernetesDir, kubeconfigphase.AdminKubeConfigFileName)) - if err != nil { - return err - } - - if err := kubemaster.UpdateMasterRoleLabelsAndTaints(client, false); err != nil { - return err - } - - // Temporary control plane is up, now we create our self hosted control - // plane components and remove the static manifests: - fmt.Println("[init] Creating self-hosted control plane...") - if err := kubemaster.CreateSelfHostedControlPlane(i.cfg, client); err != nil { - return err - } - - if i.cfg.Discovery.Token != nil { - fmt.Printf("[token-discovery] Using token: %s\n", kubeadmutil.BearerToken(i.cfg.Discovery.Token)) - if err := kubemaster.CreateDiscoveryDeploymentAndSecret(i.cfg, client, caCert); err != nil { - return err - } - if err := kubeadmutil.UpdateOrCreateToken(client, i.cfg.Discovery.Token, kubeadmutil.DefaultTokenDuration); err != nil { + // Is deployment type self-hosted? + if i.selfHosted { + // Temporary control plane is up, now we create our self hosted control + // plane components and remove the static manifests: + fmt.Println("[init] Creating self-hosted control plane...") + if err := kubemaster.CreateSelfHostedControlPlane(i.cfg, client); err != nil { return err } } diff --git a/hack/verify-flags/known-flags.txt b/hack/verify-flags/known-flags.txt index 124127c4ce2..9858f0c701f 100644 --- a/hack/verify-flags/known-flags.txt +++ b/hack/verify-flags/known-flags.txt @@ -542,6 +542,7 @@ seccomp-profile-root secondary-node-eviction-rate secret-name secure-port +self-hosted serialize-image-pulls server-start-timeout service-account-key-file