From 740a78b0f3e5952c1cb2b15ba86f42c1c3c3a8fe Mon Sep 17 00:00:00 2001 From: fabriziopandini Date: Mon, 14 Aug 2017 16:31:09 +0200 Subject: [PATCH] Main work -- move etcd to separate phase and hook up most things --- .../app/phases/controlplane/manifests.go | 181 +++------- .../app/phases/controlplane/manifests_test.go | 333 ++++-------------- .../app/phases/controlplane/volumes.go | 29 +- .../app/phases/controlplane/volumes_test.go | 71 ---- cmd/kubeadm/app/phases/etcd/local.go | 65 ++++ cmd/kubeadm/app/phases/etcd/local_test.go | 125 +++++++ 6 files changed, 321 insertions(+), 483 deletions(-) create mode 100644 cmd/kubeadm/app/phases/etcd/local.go create mode 100644 cmd/kubeadm/app/phases/etcd/local_test.go diff --git a/cmd/kubeadm/app/phases/controlplane/manifests.go b/cmd/kubeadm/app/phases/controlplane/manifests.go index 5b019a9393e..ed073cedd5f 100644 --- a/cmd/kubeadm/app/phases/controlplane/manifests.go +++ b/cmd/kubeadm/app/phases/controlplane/manifests.go @@ -17,26 +17,19 @@ limitations under the License. package controlplane import ( - "bytes" "fmt" "os" "path/filepath" "strings" - "github.com/ghodss/yaml" - "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apimachinery/pkg/util/sets" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmapiext "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha1" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" "k8s.io/kubernetes/cmd/kubeadm/app/images" + staticpodutil "k8s.io/kubernetes/cmd/kubeadm/app/util/staticpod" authzmodes "k8s.io/kubernetes/pkg/kubeapiserver/authorizer/modes" - cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" - kubetypes "k8s.io/kubernetes/pkg/kubelet/types" "k8s.io/kubernetes/pkg/util/version" ) @@ -47,125 +40,96 @@ const ( defaultv17AdmissionControl = "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,ResourceQuota" ) -// WriteStaticPodManifests builds manifest objects based on user provided configuration and then dumps it to disk -// where kubelet will pick and schedule them. -func WriteStaticPodManifests(cfg *kubeadmapi.MasterConfiguration, k8sVersion *version.Version, manifestsDir string) error { +// CreateInitStaticPodManifestFiles will write all static pod manifest files needed to bring up the control plane. +func CreateInitStaticPodManifestFiles(manifestDir string, cfg *kubeadmapi.MasterConfiguration) error { + return createStaticPodFiles(manifestDir, cfg, kubeadmconstants.KubeAPIServer, kubeadmconstants.KubeControllerManager, kubeadmconstants.KubeScheduler) +} + +// CreateAPIServerStaticPodManifestFile will write APIserver static pod manifest file. +func CreateAPIServerStaticPodManifestFile(manifestDir string, cfg *kubeadmapi.MasterConfiguration) error { + return createStaticPodFiles(manifestDir, cfg, kubeadmconstants.KubeAPIServer) +} + +// CreateControllerManagerStaticPodManifestFile will write controller manager static pod manifest file. +func CreateControllerManagerStaticPodManifestFile(manifestDir string, cfg *kubeadmapi.MasterConfiguration) error { + return createStaticPodFiles(manifestDir, cfg, kubeadmconstants.KubeControllerManager) +} + +// CreateSchedulerStaticPodManifestFile will write scheduler static pod manifest file. +func CreateSchedulerStaticPodManifestFile(manifestDir string, cfg *kubeadmapi.MasterConfiguration) error { + return createStaticPodFiles(manifestDir, cfg, kubeadmconstants.KubeScheduler) +} + +// GetStaticPodSpecs returns all staticPodSpecs actualized to the context of the current MasterConfiguration +// NB. this methods holds the information about how kubeadm creates static pod mainfests. +func GetStaticPodSpecs(cfg *kubeadmapi.MasterConfiguration, k8sVersion *version.Version) map[string]v1.Pod { // Get the required hostpath mounts mounts := getHostPathVolumesForTheControlPlane(cfg) // Prepare static pod specs staticPodSpecs := map[string]v1.Pod{ - kubeadmconstants.KubeAPIServer: componentPod(v1.Container{ + kubeadmconstants.KubeAPIServer: staticpodutil.ComponentPod(v1.Container{ Name: kubeadmconstants.KubeAPIServer, Image: images.GetCoreImage(kubeadmconstants.KubeAPIServer, cfg.ImageRepository, cfg.KubernetesVersion, cfg.UnifiedControlPlaneImage), Command: getAPIServerCommand(cfg, k8sVersion), VolumeMounts: mounts.GetVolumeMounts(kubeadmconstants.KubeAPIServer), - LivenessProbe: componentProbe(int(cfg.API.BindPort), "/healthz", v1.URISchemeHTTPS), - Resources: componentResources("250m"), + LivenessProbe: staticpodutil.ComponentProbe(int(cfg.API.BindPort), "/healthz", v1.URISchemeHTTPS), + Resources: staticpodutil.ComponentResources("250m"), Env: getProxyEnvVars(), }, mounts.GetVolumes(kubeadmconstants.KubeAPIServer)), - kubeadmconstants.KubeControllerManager: componentPod(v1.Container{ + kubeadmconstants.KubeControllerManager: staticpodutil.ComponentPod(v1.Container{ Name: kubeadmconstants.KubeControllerManager, Image: images.GetCoreImage(kubeadmconstants.KubeControllerManager, cfg.ImageRepository, cfg.KubernetesVersion, cfg.UnifiedControlPlaneImage), Command: getControllerManagerCommand(cfg, k8sVersion), VolumeMounts: mounts.GetVolumeMounts(kubeadmconstants.KubeControllerManager), - LivenessProbe: componentProbe(10252, "/healthz", v1.URISchemeHTTP), - Resources: componentResources("200m"), + LivenessProbe: staticpodutil.ComponentProbe(10252, "/healthz", v1.URISchemeHTTP), + Resources: staticpodutil.ComponentResources("200m"), Env: getProxyEnvVars(), }, mounts.GetVolumes(kubeadmconstants.KubeControllerManager)), - kubeadmconstants.KubeScheduler: componentPod(v1.Container{ + kubeadmconstants.KubeScheduler: staticpodutil.ComponentPod(v1.Container{ Name: kubeadmconstants.KubeScheduler, Image: images.GetCoreImage(kubeadmconstants.KubeScheduler, cfg.ImageRepository, cfg.KubernetesVersion, cfg.UnifiedControlPlaneImage), Command: getSchedulerCommand(cfg), VolumeMounts: mounts.GetVolumeMounts(kubeadmconstants.KubeScheduler), - LivenessProbe: componentProbe(10251, "/healthz", v1.URISchemeHTTP), - Resources: componentResources("100m"), + LivenessProbe: staticpodutil.ComponentProbe(10251, "/healthz", v1.URISchemeHTTP), + Resources: staticpodutil.ComponentResources("100m"), Env: getProxyEnvVars(), }, mounts.GetVolumes(kubeadmconstants.KubeScheduler)), } - // Add etcd static pod spec only if external etcd is not configured - if len(cfg.Etcd.Endpoints) == 0 { + return staticPodSpecs +} - etcdPod := componentPod(v1.Container{ - Name: kubeadmconstants.Etcd, - Command: getEtcdCommand(cfg), - Image: images.GetCoreImage(kubeadmconstants.Etcd, cfg.ImageRepository, "", cfg.Etcd.Image), - // Mount the etcd datadir path read-write so etcd can store data in a more persistent manner - VolumeMounts: []v1.VolumeMount{newVolumeMount(etcdVolumeName, cfg.Etcd.DataDir, false)}, - LivenessProbe: componentProbe(2379, "/health", v1.URISchemeHTTP), - }, []v1.Volume{newVolume(etcdVolumeName, cfg.Etcd.DataDir)}) +// createStaticPodFiles creates all the requested static pod files. +func createStaticPodFiles(manifestDir string, cfg *kubeadmapi.MasterConfiguration, componentNames ...string) error { - staticPodSpecs[kubeadmconstants.Etcd] = etcdPod + // TODO: Move the "pkg/util/version".Version object into the internal API instead of always parsing the string + k8sVersion, err := version.ParseSemantic(cfg.KubernetesVersion) + if err != nil { + return err } - if err := os.MkdirAll(manifestsDir, 0700); err != nil { - return fmt.Errorf("failed to create directory %q [%v]", manifestsDir, err) - } - for name, spec := range staticPodSpecs { - filename := kubeadmconstants.GetStaticPodFilepath(name, manifestsDir) - serialized, err := yaml.Marshal(spec) - if err != nil { - return fmt.Errorf("failed to marshal manifest for %q to YAML [%v]", name, err) + // gets the StaticPodSpecs, actualized for the current MasterConfiguration + specs := GetStaticPodSpecs(cfg, k8sVersion) + + // creates required static pod specs + for _, componentName := range componentNames { + // retrives the StaticPodSpec for given component + spec, exists := specs[componentName] + if !exists { + return fmt.Errorf("couldn't retrive StaticPodSpec for %s", componentName) } - if err := cmdutil.DumpReaderToFile(bytes.NewReader(serialized), filename); err != nil { - return fmt.Errorf("failed to create static pod manifest file for %q (%q) [%v]", name, filename, err) + + // writes the StaticPodSpec to disk + if err := staticpodutil.WriteStaticPodToDisk(componentName, manifestDir, spec); err != nil { + return fmt.Errorf("failed to create static pod manifest file for %q: %v", componentName, err) } } + return nil } -// componentResources returns the v1.ResourceRequirements object needed for allocating a specified amount of the CPU -func componentResources(cpu string) v1.ResourceRequirements { - return v1.ResourceRequirements{ - Requests: v1.ResourceList{ - v1.ResourceName(v1.ResourceCPU): resource.MustParse(cpu), - }, - } -} - -// componentProbe is a helper function building a ready v1.Probe object from some simple parameters -func componentProbe(port int, path string, scheme v1.URIScheme) *v1.Probe { - return &v1.Probe{ - Handler: v1.Handler{ - HTTPGet: &v1.HTTPGetAction{ - // Host has to be set to "127.0.0.1" here due to that our static Pods are on the host's network - Host: "127.0.0.1", - Path: path, - Port: intstr.FromInt(port), - Scheme: scheme, - }, - }, - InitialDelaySeconds: 15, - TimeoutSeconds: 15, - FailureThreshold: 8, - } -} - -// componentPod returns a Pod object from the container and volume specifications -func componentPod(container v1.Container, volumes []v1.Volume) v1.Pod { - return v1.Pod{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "Pod", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: container.Name, - Namespace: metav1.NamespaceSystem, - Annotations: map[string]string{kubetypes.CriticalPodAnnotationKey: ""}, - // The component and tier labels are useful for quickly identifying the control plane Pods when doing a .List() - // against Pods in the kube-system namespace. Can for example be used together with the WaitForPodsWithLabel function - Labels: map[string]string{"component": container.Name, "tier": "control-plane"}, - }, - Spec: v1.PodSpec{ - Containers: []v1.Container{container}, - HostNetwork: true, - Volumes: volumes, - }, - } -} - // getAPIServerCommand builds the right API server command from the given config object and version func getAPIServerCommand(cfg *kubeadmapi.MasterConfiguration, k8sVersion *version.Version) []string { defaultArguments := map[string]string{ @@ -195,7 +159,7 @@ func getAPIServerCommand(cfg *kubeadmapi.MasterConfiguration, k8sVersion *versio } command := []string{"kube-apiserver"} - command = append(command, getExtraParameters(cfg.APIServerExtraArgs, defaultArguments)...) + command = append(command, staticpodutil.GetExtraParameters(cfg.APIServerExtraArgs, defaultArguments)...) command = append(command, getAuthzParameters(cfg.AuthorizationModes)...) // Check if the user decided to use an external etcd cluster @@ -227,19 +191,6 @@ func getAPIServerCommand(cfg *kubeadmapi.MasterConfiguration, k8sVersion *versio return command } -// getEtcdCommand builds the right etcd command from the given config object -func getEtcdCommand(cfg *kubeadmapi.MasterConfiguration) []string { - defaultArguments := map[string]string{ - "listen-client-urls": "http://127.0.0.1:2379", - "advertise-client-urls": "http://127.0.0.1:2379", - "data-dir": cfg.Etcd.DataDir, - } - - command := []string{"etcd"} - command = append(command, getExtraParameters(cfg.Etcd.ExtraArgs, defaultArguments)...) - return command -} - // getControllerManagerCommand builds the right controller manager command from the given config object and version func getControllerManagerCommand(cfg *kubeadmapi.MasterConfiguration, k8sVersion *version.Version) []string { defaultArguments := map[string]string{ @@ -255,7 +206,7 @@ func getControllerManagerCommand(cfg *kubeadmapi.MasterConfiguration, k8sVersion } command := []string{"kube-controller-manager"} - command = append(command, getExtraParameters(cfg.ControllerManagerExtraArgs, defaultArguments)...) + command = append(command, staticpodutil.GetExtraParameters(cfg.ControllerManagerExtraArgs, defaultArguments)...) if cfg.CloudProvider != "" { command = append(command, "--cloud-provider="+cfg.CloudProvider) @@ -283,7 +234,7 @@ func getSchedulerCommand(cfg *kubeadmapi.MasterConfiguration) []string { } command := []string{"kube-scheduler"} - command = append(command, getExtraParameters(cfg.SchedulerExtraArgs, defaultArguments)...) + command = append(command, staticpodutil.GetExtraParameters(cfg.SchedulerExtraArgs, defaultArguments)...) return command } @@ -327,19 +278,3 @@ func getAuthzParameters(modes []string) []string { command = append(command, "--authorization-mode="+strings.Join(modes, ",")) return command } - -// getExtraParameters builds a list of flag arguments two string-string maps, one with default, base commands and one with overrides -func getExtraParameters(overrides map[string]string, defaults map[string]string) []string { - var command []string - for k, v := range overrides { - if len(v) > 0 { - command = append(command, fmt.Sprintf("--%s=%s", k, v)) - } - } - for k, v := range defaults { - if _, overrideExists := overrides[k]; !overrideExists { - command = append(command, fmt.Sprintf("--%s=%s", k, v)) - } - } - return command -} diff --git a/cmd/kubeadm/app/phases/controlplane/manifests_test.go b/cmd/kubeadm/app/phases/controlplane/manifests_test.go index b84bf5ac02e..23dcc03bf80 100644 --- a/cmd/kubeadm/app/phases/controlplane/manifests_test.go +++ b/cmd/kubeadm/app/phases/controlplane/manifests_test.go @@ -18,20 +18,17 @@ package controlplane import ( "fmt" - "io/ioutil" "os" "path/filepath" "reflect" "sort" "testing" - "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/intstr" - "k8s.io/apimachinery/pkg/util/yaml" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" "k8s.io/kubernetes/pkg/util/version" + + testutil "k8s.io/kubernetes/cmd/kubeadm/test" ) const ( @@ -39,189 +36,98 @@ const ( etcdDataDir = "/var/lib/etcd" ) -func TestWriteStaticPodManifests(t *testing.T) { - tmpdir, err := ioutil.TempDir("", "") - if err != nil { - t.Fatalf("Couldn't create tmpdir") +func TestGetStaticPodSpecs(t *testing.T) { + + // Creates a Master Configuration + cfg := &kubeadmapi.MasterConfiguration{ + KubernetesVersion: "v1.7.0", } - defer os.RemoveAll(tmpdir) - // set up tmp KubernetesDir for testing - kubeadmconstants.KubernetesDir = fmt.Sprintf("%s/etc/kubernetes", tmpdir) - defer func() { kubeadmconstants.KubernetesDir = "/etc/kubernetes" }() + // Executes GetStaticPodSpecs - var tests = []struct { - cfg *kubeadmapi.MasterConfiguration - expectErr bool - expectedAPIProbePort int32 + // TODO: Move the "pkg/util/version".Version object into the internal API instead of always parsing the string + k8sVersion, _ := version.ParseSemantic(cfg.KubernetesVersion) + + specs := GetStaticPodSpecs(cfg, k8sVersion) + + var assertions = []struct { + staticPodName string }{ { - cfg: &kubeadmapi.MasterConfiguration{ - KubernetesVersion: "v1.7.0", - }, - expectErr: false, + staticPodName: kubeadmconstants.KubeAPIServer, }, { - cfg: &kubeadmapi.MasterConfiguration{ - API: kubeadmapi.API{ - BindPort: 443, - }, - KubernetesVersion: "v1.7.0", - }, - expectErr: false, - expectedAPIProbePort: 443, + staticPodName: kubeadmconstants.KubeControllerManager, + }, + { + staticPodName: kubeadmconstants.KubeScheduler, }, } - for _, rt := range tests { - actual := WriteStaticPodManifests(rt.cfg, version.MustParseSemantic(rt.cfg.KubernetesVersion), fmt.Sprintf("%s/etc/kubernetes/manifests", tmpdir)) - if (actual == nil) && rt.expectErr { - t.Error("expected an error from WriteStaticPodManifests but got none") - continue - } - if (actual != nil) && !rt.expectErr { - t.Errorf("didn't expect an error from WriteStaticPodManifests but got: %v", err) - continue - } - if rt.expectErr { - continue - } + for _, assertion := range assertions { - // Below is dead code. - if rt.expectedAPIProbePort != 0 { - manifest, err := os.Open(filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.ManifestsSubDirName, "kube-apiserver.yaml")) - if err != nil { - t.Errorf("WriteStaticPodManifests: %v", err) - continue - } - defer manifest.Close() + // assert the spec for the staticPodName exists + if spec, ok := specs[assertion.staticPodName]; ok { - var pod v1.Pod - d := yaml.NewYAMLOrJSONDecoder(manifest, 4096) - if err := d.Decode(&pod); err != nil { - t.Error("WriteStaticPodManifests: error decoding manifests/kube-apiserver.yaml into Pod") - continue + // Assert each specs refers to the right pod + if spec.Spec.Containers[0].Name != assertion.staticPodName { + t.Errorf("getKubeConfigSpecs spec for %s contains pod %s, expectes %s", assertion.staticPodName, spec.Spec.Containers[0].Name, assertion.staticPodName) } - // Lots of individual checks as we traverse pointers so we don't panic dereferencing a nil on failure - containers := pod.Spec.Containers - if containers == nil || len(containers) == 0 { - t.Error("WriteStaticPodManifests: wrote an apiserver manifest without any containers") - continue - } - - probe := containers[0].LivenessProbe - if probe == nil { - t.Error("WriteStaticPodManifests: wrote an apiserver manifest without a liveness probe") - continue - } - - httpGET := probe.Handler.HTTPGet - if httpGET == nil { - t.Error("WriteStaticPodManifests: wrote an apiserver manifest without an HTTP liveness probe") - continue - } - - port := httpGET.Port.IntVal - if rt.expectedAPIProbePort != port { - t.Errorf("WriteStaticPodManifests: apiserver pod liveness probe port was: %v, wanted %v", port, rt.expectedAPIProbePort) - } + } else { + t.Errorf("getStaticPodSpecs didn't create spec for %s ", assertion.staticPodName) } } } -func TestComponentResources(t *testing.T) { - a := componentResources("250m") - if a.Requests == nil { - t.Errorf( - "failed componentResources, return value was nil", - ) - } -} +func TestCreateStaticPodFilesAndWrappers(t *testing.T) { -func TestComponentProbe(t *testing.T) { var tests = []struct { - port int - path string - scheme v1.URIScheme + createStaticPodFunction func(outDir string, cfg *kubeadmapi.MasterConfiguration) error + expectedFiles []string }{ - { - port: 1, - path: "foo", - scheme: v1.URISchemeHTTP, + { // CreateInitStaticPodManifestFiles + createStaticPodFunction: CreateInitStaticPodManifestFiles, + expectedFiles: []string{kubeadmconstants.KubeAPIServer, kubeadmconstants.KubeControllerManager, kubeadmconstants.KubeScheduler}, }, - { - port: 2, - path: "bar", - scheme: v1.URISchemeHTTPS, + { // CreateAPIServerStaticPodManifestFile + createStaticPodFunction: CreateAPIServerStaticPodManifestFile, + expectedFiles: []string{kubeadmconstants.KubeAPIServer}, }, - } - for _, rt := range tests { - actual := componentProbe(rt.port, rt.path, rt.scheme) - if actual.Handler.HTTPGet.Port != intstr.FromInt(rt.port) { - t.Errorf( - "failed componentProbe:\n\texpected: %v\n\t actual: %v", - rt.port, - actual.Handler.HTTPGet.Port, - ) - } - if actual.Handler.HTTPGet.Path != rt.path { - t.Errorf( - "failed componentProbe:\n\texpected: %s\n\t actual: %s", - rt.path, - actual.Handler.HTTPGet.Path, - ) - } - if actual.Handler.HTTPGet.Scheme != rt.scheme { - t.Errorf( - "failed componentProbe:\n\texpected: %v\n\t actual: %v", - rt.scheme, - actual.Handler.HTTPGet.Scheme, - ) - } - } -} - -func TestComponentPod(t *testing.T) { - var tests = []struct { - name string - expected v1.Pod - }{ - { - name: "foo", - expected: v1.Pod{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "Pod", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - Namespace: "kube-system", - Annotations: map[string]string{"scheduler.alpha.kubernetes.io/critical-pod": ""}, - Labels: map[string]string{"component": "foo", "tier": "control-plane"}, - }, - Spec: v1.PodSpec{ - Containers: []v1.Container{ - { - Name: "foo", - }, - }, - HostNetwork: true, - Volumes: []v1.Volume{}, - }, - }, + { // CreateControllerManagerStaticPodManifestFile + createStaticPodFunction: CreateControllerManagerStaticPodManifestFile, + expectedFiles: []string{kubeadmconstants.KubeControllerManager}, + }, + { // CreateSchedulerStaticPodManifestFile + createStaticPodFunction: CreateSchedulerStaticPodManifestFile, + expectedFiles: []string{kubeadmconstants.KubeScheduler}, }, } - for _, rt := range tests { - c := v1.Container{Name: rt.name} - actual := componentPod(c, []v1.Volume{}) - if !reflect.DeepEqual(rt.expected, actual) { - t.Errorf( - "failed componentPod:\n\texpected: %v\n\t actual: %v", - rt.expected, - actual, - ) + for _, test := range tests { + + // Create temp folder for the test case + tmpdir := testutil.SetupTempDir(t) + defer os.RemoveAll(tmpdir) + + // Creates a Master Configuration + cfg := &kubeadmapi.MasterConfiguration{ + KubernetesVersion: "v1.7.0", + } + + // Execute createStaticPodFunction + manifestPath := filepath.Join(tmpdir, kubeadmconstants.ManifestsSubDirName) + err := test.createStaticPodFunction(manifestPath, cfg) + if err != nil { + t.Errorf("Error executing createStaticPodFunction: %v", err) + continue + } + + // Assert expected files are there + testutil.AssertFilesCount(t, manifestPath, len(test.expectedFiles)) + + for _, fileName := range test.expectedFiles { + testutil.AssertFileExists(t, manifestPath, fileName+".yaml") } } } @@ -498,62 +404,6 @@ func TestGetControllerManagerCommand(t *testing.T) { } } -func TestGetEtcdCommand(t *testing.T) { - var tests = []struct { - cfg *kubeadmapi.MasterConfiguration - expected []string - }{ - { - cfg: &kubeadmapi.MasterConfiguration{ - Etcd: kubeadmapi.Etcd{DataDir: "/var/lib/etcd"}, - }, - expected: []string{ - "etcd", - "--listen-client-urls=http://127.0.0.1:2379", - "--advertise-client-urls=http://127.0.0.1:2379", - "--data-dir=/var/lib/etcd", - }, - }, - { - cfg: &kubeadmapi.MasterConfiguration{ - Etcd: kubeadmapi.Etcd{ - DataDir: "/var/lib/etcd", - ExtraArgs: map[string]string{ - "listen-client-urls": "http://10.0.1.10:2379", - "advertise-client-urls": "http://10.0.1.10:2379", - }, - }, - }, - expected: []string{ - "etcd", - "--listen-client-urls=http://10.0.1.10:2379", - "--advertise-client-urls=http://10.0.1.10:2379", - "--data-dir=/var/lib/etcd", - }, - }, - { - cfg: &kubeadmapi.MasterConfiguration{ - Etcd: kubeadmapi.Etcd{DataDir: "/etc/foo"}, - }, - expected: []string{ - "etcd", - "--listen-client-urls=http://127.0.0.1:2379", - "--advertise-client-urls=http://127.0.0.1:2379", - "--data-dir=/etc/foo", - }, - }, - } - - for _, rt := range tests { - actual := getEtcdCommand(rt.cfg) - sort.Strings(actual) - sort.Strings(rt.expected) - if !reflect.DeepEqual(actual, rt.expected) { - t.Errorf("failed getEtcdCommand:\nexpected:\n%v\nsaw:\n%v", rt.expected, actual) - } - } -} - func TestGetSchedulerCommand(t *testing.T) { var tests = []struct { cfg *kubeadmapi.MasterConfiguration @@ -651,50 +501,3 @@ func TestGetAuthzParameters(t *testing.T) { } } } - -func TestGetExtraParameters(t *testing.T) { - var tests = []struct { - overrides map[string]string - defaults map[string]string - expected []string - }{ - { - overrides: map[string]string{ - "admission-control": "NamespaceLifecycle,LimitRanger", - }, - defaults: map[string]string{ - "admission-control": "NamespaceLifecycle", - "insecure-bind-address": "127.0.0.1", - "allow-privileged": "true", - }, - expected: []string{ - "--admission-control=NamespaceLifecycle,LimitRanger", - "--insecure-bind-address=127.0.0.1", - "--allow-privileged=true", - }, - }, - { - overrides: map[string]string{ - "admission-control": "NamespaceLifecycle,LimitRanger", - }, - defaults: map[string]string{ - "insecure-bind-address": "127.0.0.1", - "allow-privileged": "true", - }, - expected: []string{ - "--admission-control=NamespaceLifecycle,LimitRanger", - "--insecure-bind-address=127.0.0.1", - "--allow-privileged=true", - }, - }, - } - - for _, rt := range tests { - actual := getExtraParameters(rt.overrides, rt.defaults) - sort.Strings(actual) - sort.Strings(rt.expected) - if !reflect.DeepEqual(actual, rt.expected) { - t.Errorf("failed getExtraParameters:\nexpected:\n%v\nsaw:\n%v", rt.expected, actual) - } - } -} diff --git a/cmd/kubeadm/app/phases/controlplane/volumes.go b/cmd/kubeadm/app/phases/controlplane/volumes.go index 14cf4c0bccb..db60823788b 100644 --- a/cmd/kubeadm/app/phases/controlplane/volumes.go +++ b/cmd/kubeadm/app/phases/controlplane/volumes.go @@ -26,11 +26,11 @@ import ( "k8s.io/apimachinery/pkg/util/sets" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" + staticpodutil "k8s.io/kubernetes/cmd/kubeadm/app/util/staticpod" ) const ( k8sCertsVolumeName = "k8s-certs" - etcdVolumeName = "etcd" caCertsVolumeName = "ca-certs" caCertsVolumePath = "/etc/ssl/certs" caCertsPkiVolumeName = "ca-certs-etc-pki" @@ -98,8 +98,8 @@ func newControlPlaneHostPathMounts() controlPlaneHostPathMounts { } func (c *controlPlaneHostPathMounts) NewHostPathMount(component, mountName, hostPath, containerPath string, readOnly bool) { - c.volumes[component] = append(c.volumes[component], newVolume(mountName, hostPath)) - c.volumeMounts[component] = append(c.volumeMounts[component], newVolumeMount(mountName, containerPath, readOnly)) + c.volumes[component] = append(c.volumes[component], staticpodutil.NewVolume(mountName, hostPath)) + c.volumeMounts[component] = append(c.volumeMounts[component], staticpodutil.NewVolumeMount(mountName, containerPath, readOnly)) } func (c *controlPlaneHostPathMounts) AddHostPathMounts(component string, vols []v1.Volume, volMounts []v1.VolumeMount) { @@ -115,25 +115,6 @@ func (c *controlPlaneHostPathMounts) GetVolumeMounts(component string) []v1.Volu return c.volumeMounts[component] } -// newVolume creates a v1.Volume with a hostPath mount to the specified location -func newVolume(name, path string) v1.Volume { - return v1.Volume{ - Name: name, - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{Path: path}, - }, - } -} - -// newVolumeMount creates a v1.VolumeMount to the specified location -func newVolumeMount(name, path string, readOnly bool) v1.VolumeMount { - return v1.VolumeMount{ - Name: name, - MountPath: path, - ReadOnly: readOnly, - } -} - // getEtcdCertVolumes returns the volumes/volumemounts needed for talking to an external etcd cluster func getEtcdCertVolumes(etcdCfg kubeadmapi.Etcd) ([]v1.Volume, []v1.VolumeMount) { certPaths := []string{etcdCfg.CAFile, etcdCfg.CertFile, etcdCfg.KeyFile} @@ -166,8 +147,8 @@ func getEtcdCertVolumes(etcdCfg kubeadmapi.Etcd) ([]v1.Volume, []v1.VolumeMount) volumeMounts := []v1.VolumeMount{} for i, certDir := range certDirs.List() { name := fmt.Sprintf("etcd-certs-%d", i) - volumes = append(volumes, newVolume(name, certDir)) - volumeMounts = append(volumeMounts, newVolumeMount(name, certDir, true)) + volumes = append(volumes, staticpodutil.NewVolume(name, certDir)) + volumeMounts = append(volumeMounts, staticpodutil.NewVolumeMount(name, certDir, true)) } return volumes, volumeMounts } diff --git a/cmd/kubeadm/app/phases/controlplane/volumes_test.go b/cmd/kubeadm/app/phases/controlplane/volumes_test.go index 81f58a28d16..d711e03bad1 100644 --- a/cmd/kubeadm/app/phases/controlplane/volumes_test.go +++ b/cmd/kubeadm/app/phases/controlplane/volumes_test.go @@ -28,77 +28,6 @@ import ( kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" ) -func TestNewVolume(t *testing.T) { - var tests = []struct { - name string - path string - expected v1.Volume - }{ - { - name: "foo", - path: "/etc/foo", - expected: v1.Volume{ - Name: "foo", - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{Path: "/etc/foo"}, - }, - }, - }, - } - - for _, rt := range tests { - actual := newVolume(rt.name, rt.path) - if !reflect.DeepEqual(actual, rt.expected) { - t.Errorf( - "failed newVolume:\n\texpected: %v\n\t actual: %v", - rt.expected, - actual, - ) - } - } -} - -func TestNewVolumeMount(t *testing.T) { - var tests = []struct { - name string - path string - ro bool - expected v1.VolumeMount - }{ - { - name: "foo", - path: "/etc/foo", - ro: false, - expected: v1.VolumeMount{ - Name: "foo", - MountPath: "/etc/foo", - ReadOnly: false, - }, - }, - { - name: "bar", - path: "/etc/foo/bar", - ro: true, - expected: v1.VolumeMount{ - Name: "bar", - MountPath: "/etc/foo/bar", - ReadOnly: true, - }, - }, - } - - for _, rt := range tests { - actual := newVolumeMount(rt.name, rt.path, rt.ro) - if !reflect.DeepEqual(actual, rt.expected) { - t.Errorf( - "failed newVolumeMount:\n\texpected: %v\n\t actual: %v", - rt.expected, - actual, - ) - } - } -} - func TestGetEtcdCertVolumes(t *testing.T) { var tests = []struct { ca, cert, key string diff --git a/cmd/kubeadm/app/phases/etcd/local.go b/cmd/kubeadm/app/phases/etcd/local.go new file mode 100644 index 00000000000..17f2af83814 --- /dev/null +++ b/cmd/kubeadm/app/phases/etcd/local.go @@ -0,0 +1,65 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package etcd + +import ( + "k8s.io/api/core/v1" + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" + kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" + "k8s.io/kubernetes/cmd/kubeadm/app/images" + staticpodutil "k8s.io/kubernetes/cmd/kubeadm/app/util/staticpod" +) + +const ( + etcdVolumeName = "etcd" +) + +// CreateLocalEtcdStaticPodManifestFile will write local etcd static pod manifest file. +func CreateLocalEtcdStaticPodManifestFile(manifestDir string, cfg *kubeadmapi.MasterConfiguration) error { + + // gets etcd StaticPodSpec, actualized for the current MasterConfiguration + spec := GetEtcdPodSpec(cfg) + + // writes etcd StaticPod to disk + return staticpodutil.WriteStaticPodToDisk(kubeadmconstants.Etcd, manifestDir, spec) +} + +// GetEtcdPodSpec returns the etcd static Pod actualized to the context of the current MasterConfiguration +// NB. GetEtcdPodSpec methods holds the information about how kubeadm creates etcd static pod mainfests. +func GetEtcdPodSpec(cfg *kubeadmapi.MasterConfiguration) v1.Pod { + return staticpodutil.ComponentPod(v1.Container{ + Name: kubeadmconstants.Etcd, + Command: getEtcdCommand(cfg), + Image: images.GetCoreImage(kubeadmconstants.Etcd, cfg.ImageRepository, "", cfg.Etcd.Image), + // Mount the etcd datadir path read-write so etcd can store data in a more persistent manner + VolumeMounts: []v1.VolumeMount{staticpodutil.NewVolumeMount(etcdVolumeName, cfg.Etcd.DataDir, false)}, + LivenessProbe: staticpodutil.ComponentProbe(2379, "/health", v1.URISchemeHTTP), + }, []v1.Volume{staticpodutil.NewVolume(etcdVolumeName, cfg.Etcd.DataDir)}) +} + +// getEtcdCommand builds the right etcd command from the given config object +func getEtcdCommand(cfg *kubeadmapi.MasterConfiguration) []string { + defaultArguments := map[string]string{ + "listen-client-urls": "http://127.0.0.1:2379", + "advertise-client-urls": "http://127.0.0.1:2379", + "data-dir": cfg.Etcd.DataDir, + } + + command := []string{"etcd"} + command = append(command, staticpodutil.GetExtraParameters(cfg.Etcd.ExtraArgs, defaultArguments)...) + return command +} diff --git a/cmd/kubeadm/app/phases/etcd/local_test.go b/cmd/kubeadm/app/phases/etcd/local_test.go new file mode 100644 index 00000000000..0d076ca0470 --- /dev/null +++ b/cmd/kubeadm/app/phases/etcd/local_test.go @@ -0,0 +1,125 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package etcd + +import ( + "os" + "path/filepath" + "reflect" + "sort" + "testing" + + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" + kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" + + testutil "k8s.io/kubernetes/cmd/kubeadm/test" +) + +func TestGetEtcdPodSpec(t *testing.T) { + + // Creates a Master Configuration + cfg := &kubeadmapi.MasterConfiguration{ + KubernetesVersion: "v1.7.0", + } + + // Executes GetEtcdPodSpec + spec := GetEtcdPodSpec(cfg) + + // Assert each specs refers to the right pod + if spec.Spec.Containers[0].Name != kubeadmconstants.Etcd { + t.Errorf("getKubeConfigSpecs spec for etcd contains pod %s, expectes %s", spec.Spec.Containers[0].Name, kubeadmconstants.Etcd) + } +} + +func TestCreateLocalEtcdStaticPodManifestFile(t *testing.T) { + + // Create temp folder for the test case + tmpdir := testutil.SetupTempDir(t) + defer os.RemoveAll(tmpdir) + + // Creates a Master Configuration + cfg := &kubeadmapi.MasterConfiguration{ + KubernetesVersion: "v1.7.0", + } + + // Execute createStaticPodFunction + manifestPath := filepath.Join(tmpdir, kubeadmconstants.ManifestsSubDirName) + err := CreateLocalEtcdStaticPodManifestFile(manifestPath, cfg) + if err != nil { + t.Errorf("Error executing CreateEtcdStaticPodManifestFile: %v", err) + } + + // Assert expected files are there + testutil.AssertFilesCount(t, manifestPath, 1) + testutil.AssertFileExists(t, manifestPath, kubeadmconstants.Etcd+".yaml") +} + +func TestGetEtcdCommand(t *testing.T) { + var tests = []struct { + cfg *kubeadmapi.MasterConfiguration + expected []string + }{ + { + cfg: &kubeadmapi.MasterConfiguration{ + Etcd: kubeadmapi.Etcd{DataDir: "/var/lib/etcd"}, + }, + expected: []string{ + "etcd", + "--listen-client-urls=http://127.0.0.1:2379", + "--advertise-client-urls=http://127.0.0.1:2379", + "--data-dir=/var/lib/etcd", + }, + }, + { + cfg: &kubeadmapi.MasterConfiguration{ + Etcd: kubeadmapi.Etcd{ + DataDir: "/var/lib/etcd", + ExtraArgs: map[string]string{ + "listen-client-urls": "http://10.0.1.10:2379", + "advertise-client-urls": "http://10.0.1.10:2379", + }, + }, + }, + expected: []string{ + "etcd", + "--listen-client-urls=http://10.0.1.10:2379", + "--advertise-client-urls=http://10.0.1.10:2379", + "--data-dir=/var/lib/etcd", + }, + }, + { + cfg: &kubeadmapi.MasterConfiguration{ + Etcd: kubeadmapi.Etcd{DataDir: "/etc/foo"}, + }, + expected: []string{ + "etcd", + "--listen-client-urls=http://127.0.0.1:2379", + "--advertise-client-urls=http://127.0.0.1:2379", + "--data-dir=/etc/foo", + }, + }, + } + + for _, rt := range tests { + actual := getEtcdCommand(rt.cfg) + sort.Strings(actual) + sort.Strings(rt.expected) + if !reflect.DeepEqual(actual, rt.expected) { + t.Errorf("failed getEtcdCommand:\nexpected:\n%v\nsaw:\n%v", rt.expected, actual) + } + } +}