diff --git a/cmd/kubeadm/app/cmd/BUILD b/cmd/kubeadm/app/cmd/BUILD index bc5ecf0a578..884451e5d8b 100644 --- a/cmd/kubeadm/app/cmd/BUILD +++ b/cmd/kubeadm/app/cmd/BUILD @@ -24,13 +24,12 @@ go_library( deps = [ "//cmd/kubeadm/app/apis/kubeadm:go_default_library", "//cmd/kubeadm/app/apis/kubeadm/v1alpha1:go_default_library", + "//cmd/kubeadm/app/cmd/flags:go_default_library", "//cmd/kubeadm/app/master:go_default_library", "//cmd/kubeadm/app/node:go_default_library", "//cmd/kubeadm/app/preflight:go_default_library", "//cmd/kubeadm/app/util:go_default_library", "//pkg/api:go_default_library", - "//pkg/cloudprovider:go_default_library", - "//pkg/cloudprovider/providers:go_default_library", "//pkg/kubectl/cmd/util:go_default_library", "//pkg/runtime:go_default_library", "//pkg/util/flag:go_default_library", diff --git a/cmd/kubeadm/app/cmd/flags/BUILD b/cmd/kubeadm/app/cmd/flags/BUILD new file mode 100644 index 00000000000..2f28f57efca --- /dev/null +++ b/cmd/kubeadm/app/cmd/flags/BUILD @@ -0,0 +1,18 @@ +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) + +load( + "@io_bazel_rules_go//go:def.bzl", + "go_binary", + "go_library", + "go_test", + "cgo_library", +) + +go_library( + name = "go_default_library", + srcs = ["cloudprovider.go"], + tags = ["automanaged"], + deps = ["//vendor:github.com/spf13/pflag"], +) diff --git a/cmd/kubeadm/app/cmd/flags/cloudprovider.go b/cmd/kubeadm/app/cmd/flags/cloudprovider.go new file mode 100644 index 00000000000..0058344fb79 --- /dev/null +++ b/cmd/kubeadm/app/cmd/flags/cloudprovider.go @@ -0,0 +1,70 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package flags + +import ( + "fmt" + + "github.com/spf13/pflag" +) + +var cloudproviders = []string{ + "aws", + "azure", + "cloudstack", + "gce", + "mesos", + "openstack", + "ovirt", + "photon", + "rackspace", + "vsphere", +} + +func NewCloudProviderFlag(provider *string) pflag.Value { + return &cloudProviderValue{provider: provider} +} + +type cloudProviderValue struct { + provider *string +} + +func (c *cloudProviderValue) String() string { + return *c.provider +} + +func (c *cloudProviderValue) Set(s string) error { + if ValidateCloudProvider(s) { + *c.provider = s + return nil + } + + return fmt.Errorf("cloud provider %q is not supported, you can use any of %v", s, cloudproviders) +} + +func (c *cloudProviderValue) Type() string { + return "cloudprovider" +} + +func ValidateCloudProvider(provider string) bool { + for _, supported := range cloudproviders { + if provider == supported { + return true + } + } + return false +} diff --git a/cmd/kubeadm/app/cmd/init.go b/cmd/kubeadm/app/cmd/init.go index aa889816701..ec070e2bd1c 100644 --- a/cmd/kubeadm/app/cmd/init.go +++ b/cmd/kubeadm/app/cmd/init.go @@ -28,12 +28,11 @@ import ( kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmapiext "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha1" + "k8s.io/kubernetes/cmd/kubeadm/app/cmd/flags" kubemaster "k8s.io/kubernetes/cmd/kubeadm/app/master" "k8s.io/kubernetes/cmd/kubeadm/app/preflight" kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" "k8s.io/kubernetes/pkg/api" - "k8s.io/kubernetes/pkg/cloudprovider" - _ "k8s.io/kubernetes/pkg/cloudprovider/providers" "k8s.io/kubernetes/pkg/runtime" netutil "k8s.io/kubernetes/pkg/util/net" ) @@ -103,9 +102,9 @@ func NewCmdInit(out io.Writer) *cobra.Command { &cfg.Networking.DNSDomain, "service-dns-domain", cfg.Networking.DNSDomain, `Use alternative domain for services, e.g. "myorg.internal"`, ) - cmd.PersistentFlags().StringVar( - &cfg.CloudProvider, "cloud-provider", cfg.CloudProvider, - `Enable cloud provider features (external load-balancers, storage, etc), e.g. "gce"`, + cmd.PersistentFlags().Var( + flags.NewCloudProviderFlag(&cfg.CloudProvider), "cloud-provider", + `Enable cloud provider features (external load-balancers, storage, etc). Note that you have to configure all kubelets manually`, ) cmd.PersistentFlags().StringVar( @@ -205,14 +204,12 @@ func NewInit(cfgPath string, cfg *kubeadmapi.MasterConfiguration, skipPreFlight cfg.KubernetesVersion = ver fmt.Println("Using Kubernetes version:", ver) - // TODO(phase1+) create a custom flag + // Warn about the limitations with the current cloudprovider solution. if cfg.CloudProvider != "" { - if cloudprovider.IsCloudProvider(cfg.CloudProvider) { - fmt.Printf("cloud provider %q initialized for the control plane. Remember to set the same cloud provider flag on the kubelet.\n", cfg.CloudProvider) - } else { - return nil, fmt.Errorf("cloud provider %q is not supported, you can use any of %v, or leave it unset.\n", cfg.CloudProvider, cloudprovider.CloudProviders()) - } + fmt.Println("WARNING: For cloudprovider integrations to work --cloud-provider must be set for all kubelets in the cluster.") + fmt.Println("\t(/etc/systemd/system/kubelet.service.d/10-kubeadm.conf should be edited for this purpose)") } + return &Init{cfg: cfg}, nil } diff --git a/cmd/kubeadm/app/images/images.go b/cmd/kubeadm/app/images/images.go index 638d4afc54e..4a6d8bb9b18 100644 --- a/cmd/kubeadm/app/images/images.go +++ b/cmd/kubeadm/app/images/images.go @@ -31,18 +31,20 @@ const ( KubeSchedulerImage = "scheduler" KubeProxyImage = "proxy" - KubeDNSImage = "kube-dns" - KubeDNSmasqImage = "dnsmasq" - KubeExechealthzImage = "exechealthz" - Pause = "pause" + KubeDNSImage = "kubedns" + KubeDNSmasqImage = "kube-dnsmasq" + KubeDNSmasqMetricsImage = "dnsmasq-metrics" + KubeExechealthzImage = "exechealthz" + Pause = "pause" gcrPrefix = "gcr.io/google_containers" etcdVersion = "3.0.14-kubeadm" - kubeDNSVersion = "1.7" - dnsmasqVersion = "1.3" - exechealthzVersion = "1.1" - pauseVersion = "3.0" + kubeDNSVersion = "1.9" + dnsmasqVersion = "1.4" + exechealthzVersion = "1.2" + dnsmasqMetricsVersion = "1.0" + pauseVersion = "3.0" ) func GetCoreImage(image string, cfg *kubeadmapi.MasterConfiguration, overrideImage string) string { @@ -62,9 +64,10 @@ func GetCoreImage(image string, cfg *kubeadmapi.MasterConfiguration, overrideIma func GetAddonImage(image string) string { repoPrefix := kubeadmapi.GlobalEnvParams.RepositoryPrefix return map[string]string{ - KubeDNSImage: fmt.Sprintf("%s/%s-%s:%s", repoPrefix, "kubedns", runtime.GOARCH, kubeDNSVersion), - KubeDNSmasqImage: fmt.Sprintf("%s/%s-%s:%s", repoPrefix, "kube-dnsmasq", runtime.GOARCH, dnsmasqVersion), - KubeExechealthzImage: fmt.Sprintf("%s/%s-%s:%s", repoPrefix, "exechealthz", runtime.GOARCH, exechealthzVersion), - Pause: fmt.Sprintf("%s/%s-%s:%s", repoPrefix, "pause", runtime.GOARCH, pauseVersion), + KubeDNSImage: fmt.Sprintf("%s/%s-%s:%s", repoPrefix, "kubedns", runtime.GOARCH, kubeDNSVersion), + KubeDNSmasqImage: fmt.Sprintf("%s/%s-%s:%s", repoPrefix, "kube-dnsmasq", runtime.GOARCH, dnsmasqVersion), + KubeDNSmasqMetricsImage: fmt.Sprintf("%s/%s-%s:%s", repoPrefix, "dnsmasq-metrics", runtime.GOARCH, dnsmasqMetricsVersion), + KubeExechealthzImage: fmt.Sprintf("%s/%s-%s:%s", repoPrefix, "exechealthz", runtime.GOARCH, exechealthzVersion), + Pause: fmt.Sprintf("%s/%s-%s:%s", repoPrefix, "pause", runtime.GOARCH, pauseVersion), }[image] } diff --git a/cmd/kubeadm/app/master/addons.go b/cmd/kubeadm/app/master/addons.go index ff165bbd169..c76a1563bcb 100644 --- a/cmd/kubeadm/app/master/addons.go +++ b/cmd/kubeadm/app/master/addons.go @@ -27,7 +27,7 @@ import ( "k8s.io/kubernetes/pkg/api/resource" "k8s.io/kubernetes/pkg/api/v1" clientset "k8s.io/kubernetes/pkg/client/clientset_generated/release_1_5" - ipallocator "k8s.io/kubernetes/pkg/registry/core/service/ipallocator" + "k8s.io/kubernetes/pkg/registry/core/service/ipallocator" "k8s.io/kubernetes/pkg/util/intstr" ) @@ -83,26 +83,9 @@ func createKubeProxyPodSpec(cfg *kubeadmapi.MasterConfiguration) v1.PodSpec { func createKubeDNSPodSpec(cfg *kubeadmapi.MasterConfiguration) v1.PodSpec { - dnsPodResources := v1.ResourceList{ - v1.ResourceName(v1.ResourceCPU): resource.MustParse("100m"), - v1.ResourceName(v1.ResourceMemory): resource.MustParse("170Mi"), - } - - healthzPodResources := v1.ResourceList{ - v1.ResourceName(v1.ResourceCPU): resource.MustParse("10m"), - v1.ResourceName(v1.ResourceMemory): resource.MustParse("50Mi"), - } - kubeDNSPort := int32(10053) dnsmasqPort := int32(53) - nslookup := fmt.Sprintf("nslookup kubernetes.default.svc.%s 127.0.0.1", cfg.Networking.DNSDomain) - - nslookup = fmt.Sprintf("-cmd=%s:%d >/dev/null && %s:%d >/dev/null", - nslookup, dnsmasqPort, - nslookup, kubeDNSPort, - ) - return v1.PodSpec{ Containers: []v1.Container{ // DNS server @@ -110,18 +93,18 @@ func createKubeDNSPodSpec(cfg *kubeadmapi.MasterConfiguration) v1.PodSpec { Name: "kube-dns", Image: images.GetAddonImage(images.KubeDNSImage), Resources: v1.ResourceRequirements{ - Limits: dnsPodResources, - Requests: dnsPodResources, - }, - Args: []string{ - fmt.Sprintf("--domain=%s", cfg.Networking.DNSDomain), - fmt.Sprintf("--dns-port=%d", kubeDNSPort), - // TODO __PILLAR__FEDERATIONS__DOMAIN__MAP__ + Limits: v1.ResourceList{ + v1.ResourceName(v1.ResourceMemory): resource.MustParse("170Mi"), + }, + Requests: v1.ResourceList{ + v1.ResourceName(v1.ResourceCPU): resource.MustParse("100m"), + v1.ResourceName(v1.ResourceMemory): resource.MustParse("70Mi"), + }, }, LivenessProbe: &v1.Probe{ Handler: v1.Handler{ HTTPGet: &v1.HTTPGetAction{ - Path: "/healthz", + Path: "/healthz-kubedns", Port: intstr.FromInt(8080), Scheme: v1.URISchemeHTTP, }, @@ -129,7 +112,7 @@ func createKubeDNSPodSpec(cfg *kubeadmapi.MasterConfiguration) v1.PodSpec { InitialDelaySeconds: 60, TimeoutSeconds: 5, SuccessThreshold: 1, - FailureThreshold: 1, + FailureThreshold: 5, }, // # we poll on pod startup for the Kubernetes master service and // # only setup the /readiness HTTP server once that's available. @@ -141,9 +124,20 @@ func createKubeDNSPodSpec(cfg *kubeadmapi.MasterConfiguration) v1.PodSpec { Scheme: v1.URISchemeHTTP, }, }, - InitialDelaySeconds: 30, + InitialDelaySeconds: 3, TimeoutSeconds: 5, }, + Args: []string{ + fmt.Sprintf("--domain=%s", cfg.Networking.DNSDomain), + fmt.Sprintf("--dns-port=%d", kubeDNSPort), + "--config-map=kube-dns", + }, + Env: []v1.EnvVar{ + { + Name: "PROMETHEUS_PORT", + Value: "10055", + }, + }, Ports: []v1.ContainerPort{ { ContainerPort: kubeDNSPort, @@ -155,20 +149,35 @@ func createKubeDNSPodSpec(cfg *kubeadmapi.MasterConfiguration) v1.PodSpec { Name: "dns-tcp-local", Protocol: v1.ProtocolTCP, }, + { + ContainerPort: 10055, + Name: "metrics", + Protocol: v1.ProtocolTCP, + }, }, }, // dnsmasq { Name: "dnsmasq", Image: images.GetAddonImage(images.KubeDNSmasqImage), - Resources: v1.ResourceRequirements{ - Limits: dnsPodResources, - Requests: dnsPodResources, + LivenessProbe: &v1.Probe{ + Handler: v1.Handler{ + HTTPGet: &v1.HTTPGetAction{ + Path: "/healthz-dnsmasq", + Port: intstr.FromInt(8080), + Scheme: v1.URISchemeHTTP, + }, + }, + InitialDelaySeconds: 60, + TimeoutSeconds: 5, + SuccessThreshold: 1, + FailureThreshold: 5, }, Args: []string{ "--cache-size=1000", "--no-resolv", fmt.Sprintf("--server=127.0.0.1#%d", kubeDNSPort), + "--log-facility=-", }, Ports: []v1.ContainerPort{ { @@ -182,19 +191,66 @@ func createKubeDNSPodSpec(cfg *kubeadmapi.MasterConfiguration) v1.PodSpec { Protocol: v1.ProtocolTCP, }, }, + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceName(v1.ResourceCPU): resource.MustParse("150m"), + v1.ResourceName(v1.ResourceMemory): resource.MustParse("10Mi"), + }, + }, + }, + { + Name: "dnsmasq-metrics", + Image: images.GetAddonImage(images.KubeDNSmasqMetricsImage), + LivenessProbe: &v1.Probe{ + Handler: v1.Handler{ + HTTPGet: &v1.HTTPGetAction{ + Path: "/metrics", + Port: intstr.FromInt(10054), + Scheme: v1.URISchemeHTTP, + }, + }, + InitialDelaySeconds: 60, + TimeoutSeconds: 5, + SuccessThreshold: 1, + FailureThreshold: 5, + }, + Args: []string{ + "--v=2", + "--logtostderr", + }, + Ports: []v1.ContainerPort{ + { + ContainerPort: 10054, + Name: "metrics", + Protocol: v1.ProtocolTCP, + }, + }, + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceName(v1.ResourceMemory): resource.MustParse("10Mi"), + }, + }, }, // healthz { Name: "healthz", Image: images.GetAddonImage(images.KubeExechealthzImage), Resources: v1.ResourceRequirements{ - Limits: healthzPodResources, - Requests: healthzPodResources, + Limits: v1.ResourceList{ + v1.ResourceName(v1.ResourceMemory): resource.MustParse("50Mi"), + }, + Requests: v1.ResourceList{ + v1.ResourceName(v1.ResourceCPU): resource.MustParse("10m"), + v1.ResourceName(v1.ResourceMemory): resource.MustParse("50Mi"), + }, }, Args: []string{ - nslookup, - "-port=8080", - "-quiet", + fmt.Sprintf("--cmd=nslookup kubernetes.default.svc.%s 127.0.0.1 >/dev/null", cfg.Networking.DNSDomain), + "--url=/healthz-dnsmasq", + fmt.Sprintf("--cmd=nslookup kubernetes.default.svc.%s 127.0.0.1:%d >/dev/null", cfg.Networking.DNSDomain, kubeDNSPort), + "--url=/healthz-kubedns", + "--port=8080", + "--quiet", }, Ports: []v1.ContainerPort{{ ContainerPort: 8080, @@ -204,7 +260,6 @@ func createKubeDNSPodSpec(cfg *kubeadmapi.MasterConfiguration) v1.PodSpec { }, DNSPolicy: v1.DNSDefault, } - } func createKubeDNSServiceSpec(cfg *kubeadmapi.MasterConfiguration) (*v1.ServiceSpec, error) { @@ -217,16 +272,14 @@ func createKubeDNSServiceSpec(cfg *kubeadmapi.MasterConfiguration) (*v1.ServiceS return nil, fmt.Errorf("unable to allocate IP address for kube-dns addon from the given CIDR (%q) [%v]", cfg.Networking.ServiceSubnet, err) } - svc := &v1.ServiceSpec{ + return &v1.ServiceSpec{ Selector: map[string]string{"name": "kube-dns"}, Ports: []v1.ServicePort{ {Name: "dns", Port: 53, Protocol: v1.ProtocolUDP}, {Name: "dns-tcp", Port: 53, Protocol: v1.ProtocolTCP}, }, ClusterIP: ip.String(), - } - - return svc, nil + }, nil } func CreateEssentialAddons(cfg *kubeadmapi.MasterConfiguration, client *clientset.Clientset) error { @@ -250,10 +303,11 @@ func CreateEssentialAddons(cfg *kubeadmapi.MasterConfiguration, client *clientse kubeDNSServiceSpec, err := createKubeDNSServiceSpec(cfg) if err != nil { - return fmt.Errorf(" failed creating essential kube-dns addon - %v", err) + return fmt.Errorf("failed creating essential kube-dns addon [%v]", err) } kubeDNSService := NewService("kube-dns", *kubeDNSServiceSpec) + kubeDNSService.ObjectMeta.Labels["kubernetes.io/name"] = "KubeDNS" if _, err := client.Services(api.NamespaceSystem).Create(kubeDNSService); err != nil { return fmt.Errorf(" failed creating essential kube-dns addon [%v]", err) } diff --git a/cmd/kubeadm/app/master/manifests.go b/cmd/kubeadm/app/master/manifests.go index f59e6161866..f1e187833d3 100644 --- a/cmd/kubeadm/app/master/manifests.go +++ b/cmd/kubeadm/app/master/manifests.go @@ -34,10 +34,9 @@ import ( ) // Static pod definitions in golang form are included below so that `kubeadm init` can get going. - const ( DefaultClusterName = "kubernetes" - DefaultCloudConfigPath = "/etc/kubernetes/cloud-config.json" + DefaultCloudConfigPath = "/etc/kubernetes/cloud-config" etcd = "etcd" apiServer = "apiserver" @@ -312,6 +311,15 @@ func getAPIServerCommand(cfg *kubeadmapi.MasterConfiguration) (command []string) command = append(command, etcdClientFileArg, etcdKeyFileArg) } + if cfg.CloudProvider != "" { + command = append(command, "--cloud-provider="+cfg.CloudProvider) + + // Only append the --cloud-config option if there's a such file + if _, err := os.Stat(DefaultCloudConfigPath); err == nil { + command = append(command, "--cloud-config="+DefaultCloudConfigPath) + } + } + return }