diff --git a/cmd/kubeadm/app/master/BUILD b/cmd/kubeadm/app/master/BUILD index ae668fa7182..dc0292d0b12 100644 --- a/cmd/kubeadm/app/master/BUILD +++ b/cmd/kubeadm/app/master/BUILD @@ -24,6 +24,7 @@ go_library( "//pkg/bootstrap/api:go_default_library", "//pkg/kubeapiserver/authorizer/modes:go_default_library", "//pkg/kubectl/cmd/util:go_default_library", + "//pkg/util/version:go_default_library", "//vendor:github.com/ghodss/yaml", "//vendor:k8s.io/apimachinery/pkg/api/errors", "//vendor:k8s.io/apimachinery/pkg/api/resource", @@ -43,6 +44,7 @@ go_test( tags = ["automanaged"], deps = [ "//cmd/kubeadm/app/apis/kubeadm:go_default_library", + "//pkg/util/version:go_default_library", "//vendor:k8s.io/apimachinery/pkg/util/intstr", "//vendor:k8s.io/apimachinery/pkg/util/yaml", "//vendor:k8s.io/client-go/pkg/api/v1", diff --git a/cmd/kubeadm/app/master/manifests.go b/cmd/kubeadm/app/master/manifests.go index b1f416b7c3d..d79d2a8f026 100644 --- a/cmd/kubeadm/app/master/manifests.go +++ b/cmd/kubeadm/app/master/manifests.go @@ -35,6 +35,7 @@ import ( bootstrapapi "k8s.io/kubernetes/pkg/bootstrap/api" authzmodes "k8s.io/kubernetes/pkg/kubeapiserver/authorizer/modes" cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" + "k8s.io/kubernetes/pkg/util/version" ) // Static pod definitions in golang form are included below so that `kubeadm init` can get going. @@ -52,6 +53,10 @@ const ( kubeProxy = "kube-proxy" ) +var ( + v170 = version.MustParseSemantic("v1.7.0-alpha.0") +) + // WriteStaticPodManifests builds manifest objects based on user provided configuration and then dumps it to disk // where kubelet will pick and schedule them. func WriteStaticPodManifests(cfg *kubeadmapi.MasterConfiguration) error { @@ -68,12 +73,17 @@ func WriteStaticPodManifests(cfg *kubeadmapi.MasterConfiguration) error { volumeMounts = append(volumeMounts, pkiVolumeMount()) } + k8sVersion, err := version.ParseSemantic(cfg.KubernetesVersion) + if err != nil { + return err + } + // Prepare static pod specs staticPodSpecs := map[string]api.Pod{ kubeAPIServer: componentPod(api.Container{ Name: kubeAPIServer, Image: images.GetCoreImage(images.KubeAPIServerImage, cfg, kubeadmapi.GlobalEnvParams.HyperkubeImage), - Command: getAPIServerCommand(cfg, false), + Command: getAPIServerCommand(cfg, false, k8sVersion), VolumeMounts: volumeMounts, LivenessProbe: componentProbe(int(cfg.API.BindPort), "/healthz", api.URISchemeHTTPS), Resources: componentResources("250m"), @@ -293,7 +303,7 @@ func getComponentBaseCommand(component string) []string { return []string{"kube-" + component} } -func getAPIServerCommand(cfg *kubeadmapi.MasterConfiguration, selfHosted bool) []string { +func getAPIServerCommand(cfg *kubeadmapi.MasterConfiguration, selfHosted bool, k8sVersion *version.Version) []string { var command []string // self-hosted apiserver needs to wait on a lock @@ -323,9 +333,11 @@ func getAPIServerCommand(cfg *kubeadmapi.MasterConfiguration, selfHosted bool) [ "requestheader-extra-headers-prefix": "X-Remote-Extra-", "requestheader-client-ca-file": path.Join(cfg.CertificatesDir, kubeadmconstants.FrontProxyCACertName), "requestheader-allowed-names": "front-proxy-client", + } + if k8sVersion.AtLeast(v170) { // add options which allow the kube-apiserver to act as a front-proxy to aggregated API servers - "proxy-client-cert-file": path.Join(cfg.CertificatesDir, kubeadmconstants.FrontProxyClientCertName), - "proxy-client-key-file": path.Join(cfg.CertificatesDir, kubeadmconstants.FrontProxyClientKeyName), + defaultArguments["proxy-client-cert-file"] = path.Join(cfg.CertificatesDir, kubeadmconstants.FrontProxyClientCertName) + defaultArguments["proxy-client-key-file"] = path.Join(cfg.CertificatesDir, kubeadmconstants.FrontProxyClientKeyName) } command = getComponentBaseCommand(apiServer) diff --git a/cmd/kubeadm/app/master/manifests_test.go b/cmd/kubeadm/app/master/manifests_test.go index 35b31d70d45..eb9c714e11c 100644 --- a/cmd/kubeadm/app/master/manifests_test.go +++ b/cmd/kubeadm/app/master/manifests_test.go @@ -29,6 +29,7 @@ import ( api "k8s.io/client-go/pkg/api/v1" "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" + "k8s.io/kubernetes/pkg/util/version" ) const testCertsDir = "/var/lib/certs" @@ -47,12 +48,12 @@ func TestWriteStaticPodManifests(t *testing.T) { var tests = []struct { cfg *kubeadmapi.MasterConfiguration - expected bool + expectErr bool expectedAPIProbePort int32 }{ { - cfg: &kubeadmapi.MasterConfiguration{}, - expected: true, + cfg: &kubeadmapi.MasterConfiguration{}, + expectErr: true, }, { cfg: &kubeadmapi.MasterConfiguration{ @@ -60,25 +61,29 @@ func TestWriteStaticPodManifests(t *testing.T) { BindPort: 443, }, }, - expected: true, + expectErr: true, expectedAPIProbePort: 443, }, } for _, rt := range tests { actual := WriteStaticPodManifests(rt.cfg) - if (actual == nil) != rt.expected { - t.Errorf( - "failed WriteStaticPodManifests with an error:\n\texpected: %t\n\t actual: %t", - rt.expected, - (actual == nil), - ) + if (actual == nil) && rt.expectErr { + t.Error("expected an error from WriteStaticPodManifests but got none") + continue + } + if (actual != nil) && !rt.expectErr { + t.Errorf("didn't expect an error from WriteStaticPodManifests but got: %v", err) + continue + } + if rt.expectErr { continue } + // Below is dead code. if rt.expectedAPIProbePort != 0 { - manifest, err := os.Open(fmt.Sprintf("%s/manifests/kube-apiserver.yaml", kubeadmapi.GlobalEnvParams.KubernetesDir)) + manifest, err := os.Open(kubeadmapi.GlobalEnvParams.KubernetesDir + "/manifests/kube-apiserver.yaml") if err != nil { - t.Error("WriteStaticPodManifests: error opening manifests/kube-apiserver.yaml") + t.Errorf("WriteStaticPodManifests: %v", err) continue } @@ -433,9 +438,10 @@ func TestGetAPIServerCommand(t *testing.T) { }{ { cfg: &kubeadmapi.MasterConfiguration{ - API: kubeadm.API{BindPort: 123, AdvertiseAddress: "1.2.3.4"}, - Networking: kubeadm.Networking{ServiceSubnet: "bar"}, - CertificatesDir: testCertsDir, + API: kubeadm.API{BindPort: 123, AdvertiseAddress: "1.2.3.4"}, + Networking: kubeadm.Networking{ServiceSubnet: "bar"}, + CertificatesDir: testCertsDir, + KubernetesVersion: "v1.6.0", }, expected: []string{ "kube-apiserver", @@ -453,8 +459,6 @@ func TestGetAPIServerCommand(t *testing.T) { "--storage-backend=etcd3", "--kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname", "--experimental-bootstrap-token-auth=true", - "--proxy-client-cert-file=/var/lib/certs/front-proxy-client.crt", - "--proxy-client-key-file=/var/lib/certs/front-proxy-client.key", "--requestheader-username-headers=X-Remote-User", "--requestheader-group-headers=X-Remote-Group", "--requestheader-extra-headers-prefix=X-Remote-Extra-", @@ -467,9 +471,10 @@ func TestGetAPIServerCommand(t *testing.T) { }, { cfg: &kubeadmapi.MasterConfiguration{ - API: kubeadm.API{BindPort: 123, AdvertiseAddress: "4.3.2.1"}, - Networking: kubeadm.Networking{ServiceSubnet: "bar"}, - CertificatesDir: testCertsDir, + API: kubeadm.API{BindPort: 123, AdvertiseAddress: "4.3.2.1"}, + Networking: kubeadm.Networking{ServiceSubnet: "bar"}, + CertificatesDir: testCertsDir, + KubernetesVersion: "v1.6.0", }, expected: []string{ "kube-apiserver", @@ -487,8 +492,6 @@ func TestGetAPIServerCommand(t *testing.T) { "--storage-backend=etcd3", "--kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname", "--experimental-bootstrap-token-auth=true", - "--proxy-client-cert-file=/var/lib/certs/front-proxy-client.crt", - "--proxy-client-key-file=/var/lib/certs/front-proxy-client.key", "--requestheader-username-headers=X-Remote-User", "--requestheader-group-headers=X-Remote-Group", "--requestheader-extra-headers-prefix=X-Remote-Extra-", @@ -501,10 +504,47 @@ func TestGetAPIServerCommand(t *testing.T) { }, { cfg: &kubeadmapi.MasterConfiguration{ - API: kubeadm.API{BindPort: 123, AdvertiseAddress: "4.3.2.1"}, - Networking: kubeadm.Networking{ServiceSubnet: "bar"}, - Etcd: kubeadm.Etcd{CertFile: "fiz", KeyFile: "faz"}, - CertificatesDir: testCertsDir, + API: kubeadm.API{BindPort: 123, AdvertiseAddress: "4.3.2.1"}, + Networking: kubeadm.Networking{ServiceSubnet: "bar"}, + Etcd: kubeadm.Etcd{CertFile: "fiz", KeyFile: "faz"}, + CertificatesDir: testCertsDir, + KubernetesVersion: "v1.6.0", + }, + expected: []string{ + "kube-apiserver", + "--insecure-port=0", + "--admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds", + "--service-cluster-ip-range=bar", + "--service-account-key-file=" + testCertsDir + "/sa.pub", + "--client-ca-file=" + testCertsDir + "/ca.crt", + "--tls-cert-file=" + testCertsDir + "/apiserver.crt", + "--tls-private-key-file=" + testCertsDir + "/apiserver.key", + "--kubelet-client-certificate=" + testCertsDir + "/apiserver-kubelet-client.crt", + "--kubelet-client-key=" + testCertsDir + "/apiserver-kubelet-client.key", + fmt.Sprintf("--secure-port=%d", 123), + "--allow-privileged=true", + "--storage-backend=etcd3", + "--kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname", + "--experimental-bootstrap-token-auth=true", + "--requestheader-username-headers=X-Remote-User", + "--requestheader-group-headers=X-Remote-Group", + "--requestheader-extra-headers-prefix=X-Remote-Extra-", + "--requestheader-client-ca-file=" + testCertsDir + "/front-proxy-ca.crt", + "--requestheader-allowed-names=front-proxy-client", + "--authorization-mode=RBAC", + "--advertise-address=4.3.2.1", + "--etcd-servers=http://127.0.0.1:2379", + "--etcd-certfile=fiz", + "--etcd-keyfile=faz", + }, + }, + { + cfg: &kubeadmapi.MasterConfiguration{ + API: kubeadm.API{BindPort: 123, AdvertiseAddress: "4.3.2.1"}, + Networking: kubeadm.Networking{ServiceSubnet: "bar"}, + Etcd: kubeadm.Etcd{CertFile: "fiz", KeyFile: "faz"}, + CertificatesDir: testCertsDir, + KubernetesVersion: "v1.7.0", }, expected: []string{ "kube-apiserver", @@ -539,7 +579,7 @@ func TestGetAPIServerCommand(t *testing.T) { } for _, rt := range tests { - actual := getAPIServerCommand(rt.cfg, false) + actual := getAPIServerCommand(rt.cfg, false, version.MustParseSemantic(rt.cfg.KubernetesVersion)) sort.Strings(actual) sort.Strings(rt.expected) if !reflect.DeepEqual(actual, rt.expected) { @@ -760,3 +800,18 @@ func TestGetExtraParameters(t *testing.T) { } } } + +func TestVersionCompare(t *testing.T) { + versions := []string{ + "v1.7.0-alpha.1", + "v1.7.0-beta.0", + "v1.7.0-rc.0", + "v1.7.0", + "v1.7.1", + } + for _, v := range versions { + if !version.MustParseSemantic(v).AtLeast(v170) { + t.Errorf("err") + } + } +} diff --git a/cmd/kubeadm/app/master/selfhosted.go b/cmd/kubeadm/app/master/selfhosted.go index 194e892e2a1..301caf96f3a 100644 --- a/cmd/kubeadm/app/master/selfhosted.go +++ b/cmd/kubeadm/app/master/selfhosted.go @@ -31,6 +31,7 @@ import ( kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" "k8s.io/kubernetes/cmd/kubeadm/app/images" + "k8s.io/kubernetes/pkg/util/version" ) var ( @@ -74,7 +75,11 @@ func CreateSelfHostedControlPlane(cfg *kubeadmapi.MasterConfiguration, client *c func launchSelfHostedAPIServer(cfg *kubeadmapi.MasterConfiguration, client *clientset.Clientset, volumes []v1.Volume, volumeMounts []v1.VolumeMount) error { start := time.Now() - apiServer := getAPIServerDS(cfg, volumes, volumeMounts) + kubeVersion, err := version.ParseSemantic(cfg.KubernetesVersion) + if err != nil { + return err + } + apiServer := getAPIServerDS(cfg, volumes, volumeMounts, kubeVersion) if _, err := client.Extensions().DaemonSets(metav1.NamespaceSystem).Create(&apiServer); err != nil { return fmt.Errorf("failed to create self-hosted %q daemon set [%v]", kubeAPIServer, err) } @@ -183,7 +188,7 @@ func waitForPodsWithLabel(client *clientset.Clientset, appLabel string, mustBeRu } // Sources from bootkube templates.go -func getAPIServerDS(cfg *kubeadmapi.MasterConfiguration, volumes []v1.Volume, volumeMounts []v1.VolumeMount) ext.DaemonSet { +func getAPIServerDS(cfg *kubeadmapi.MasterConfiguration, volumes []v1.Volume, volumeMounts []v1.VolumeMount, kubeVersion *version.Version) ext.DaemonSet { ds := ext.DaemonSet{ TypeMeta: metav1.TypeMeta{ APIVersion: "extensions/v1beta1", @@ -211,7 +216,7 @@ func getAPIServerDS(cfg *kubeadmapi.MasterConfiguration, volumes []v1.Volume, vo { Name: "self-hosted-" + kubeAPIServer, Image: images.GetCoreImage(images.KubeAPIServerImage, cfg, kubeadmapi.GlobalEnvParams.HyperkubeImage), - Command: getAPIServerCommand(cfg, true), + Command: getAPIServerCommand(cfg, true, kubeVersion), Env: getSelfHostedAPIServerEnv(), VolumeMounts: volumeMounts, LivenessProbe: componentProbe(6443, "/healthz", v1.URISchemeHTTPS),