From 7e82985f6d41bc55e7a463f9c13951be1437d0ca Mon Sep 17 00:00:00 2001 From: Jamie Hannaford Date: Tue, 28 Feb 2017 11:12:57 +0100 Subject: [PATCH] Allow configurable etcd options --- cmd/kubeadm/app/apis/kubeadm/env.go | 2 - cmd/kubeadm/app/apis/kubeadm/fuzzer/fuzzer.go | 1 + cmd/kubeadm/app/apis/kubeadm/types.go | 3 +- .../app/apis/kubeadm/v1alpha1/defaults.go | 5 ++ .../app/apis/kubeadm/v1alpha1/types.go | 10 +-- cmd/kubeadm/app/master/manifests.go | 34 +++++---- cmd/kubeadm/app/master/manifests_test.go | 72 +++++++++++++++++-- cmd/kubeadm/app/preflight/checks.go | 2 +- 8 files changed, 103 insertions(+), 26 deletions(-) diff --git a/cmd/kubeadm/app/apis/kubeadm/env.go b/cmd/kubeadm/app/apis/kubeadm/env.go index a40f37343cd..ab97cdbbf1a 100644 --- a/cmd/kubeadm/app/apis/kubeadm/env.go +++ b/cmd/kubeadm/app/apis/kubeadm/env.go @@ -31,7 +31,6 @@ func SetEnvParams() *EnvParams { envParams := map[string]string{ "kubernetes_dir": "/etc/kubernetes", - "host_etcd_path": "/var/lib/etcd", "hyperkube_image": "", "repo_prefix": "gcr.io/google_containers", "etcd_image": "", @@ -45,7 +44,6 @@ func SetEnvParams() *EnvParams { return &EnvParams{ KubernetesDir: path.Clean(envParams["kubernetes_dir"]), - HostEtcdPath: path.Clean(envParams["host_etcd_path"]), HyperkubeImage: envParams["hyperkube_image"], RepositoryPrefix: envParams["repo_prefix"], EtcdImage: envParams["etcd_image"], diff --git a/cmd/kubeadm/app/apis/kubeadm/fuzzer/fuzzer.go b/cmd/kubeadm/app/apis/kubeadm/fuzzer/fuzzer.go index 7d5bdac3919..ad067f46b72 100644 --- a/cmd/kubeadm/app/apis/kubeadm/fuzzer/fuzzer.go +++ b/cmd/kubeadm/app/apis/kubeadm/fuzzer/fuzzer.go @@ -36,6 +36,7 @@ func KubeadmFuzzerFuncs(t apitesting.TestingCommon) []interface{} { obj.CertificatesDir = "foo" obj.APIServerCertSANs = []string{} obj.Token = "foo" + obj.Etcd.DataDir = "foo" }, func(obj *kubeadm.NodeConfiguration, c fuzz.Continue) { c.FuzzNoCustom(obj) diff --git a/cmd/kubeadm/app/apis/kubeadm/types.go b/cmd/kubeadm/app/apis/kubeadm/types.go index 0686f302b0e..4e836969554 100644 --- a/cmd/kubeadm/app/apis/kubeadm/types.go +++ b/cmd/kubeadm/app/apis/kubeadm/types.go @@ -24,7 +24,6 @@ import ( type EnvParams struct { KubernetesDir string - HostEtcdPath string HyperkubeImage string RepositoryPrefix string EtcdImage string @@ -82,6 +81,8 @@ type Etcd struct { CAFile string CertFile string KeyFile string + DataDir string + ExtraArgs map[string]string } type NodeConfiguration struct { diff --git a/cmd/kubeadm/app/apis/kubeadm/v1alpha1/defaults.go b/cmd/kubeadm/app/apis/kubeadm/v1alpha1/defaults.go index 4ea69723992..d7fe908f442 100644 --- a/cmd/kubeadm/app/apis/kubeadm/v1alpha1/defaults.go +++ b/cmd/kubeadm/app/apis/kubeadm/v1alpha1/defaults.go @@ -32,6 +32,7 @@ const ( DefaultAuthorizationMode = "RBAC" DefaultCACertPath = "/etc/kubernetes/pki/ca.crt" DefaultCertificatesDir = "/etc/kubernetes/pki" + DefaultEtcdDataDir = "/var/lib/etcd" ) func addDefaultingFuncs(scheme *runtime.Scheme) error { @@ -70,6 +71,10 @@ func SetDefaults_MasterConfiguration(obj *MasterConfiguration) { if obj.TokenTTL == 0 { obj.TokenTTL = constants.DefaultTokenDuration } + + if obj.Etcd.DataDir == "" { + obj.Etcd.DataDir = DefaultEtcdDataDir + } } func SetDefaults_NodeConfiguration(obj *NodeConfiguration) { diff --git a/cmd/kubeadm/app/apis/kubeadm/v1alpha1/types.go b/cmd/kubeadm/app/apis/kubeadm/v1alpha1/types.go index 3fe5d7946e0..b2e6d27c2c3 100644 --- a/cmd/kubeadm/app/apis/kubeadm/v1alpha1/types.go +++ b/cmd/kubeadm/app/apis/kubeadm/v1alpha1/types.go @@ -70,10 +70,12 @@ type Networking struct { } type Etcd struct { - Endpoints []string `json:"endpoints"` - CAFile string `json:"caFile"` - CertFile string `json:"certFile"` - KeyFile string `json:"keyFile"` + Endpoints []string `json:"endpoints"` + CAFile string `json:"caFile"` + CertFile string `json:"certFile"` + KeyFile string `json:"keyFile"` + DataDir string `json:"dataDir"` + ExtraArgs map[string]string `json:"extraArgs"` } type NodeConfiguration struct { diff --git a/cmd/kubeadm/app/master/manifests.go b/cmd/kubeadm/app/master/manifests.go index e00e5994cb0..c82aded8a8e 100644 --- a/cmd/kubeadm/app/master/manifests.go +++ b/cmd/kubeadm/app/master/manifests.go @@ -102,21 +102,16 @@ func WriteStaticPodManifests(cfg *kubeadmapi.MasterConfiguration) error { // Add etcd static pod spec only if external etcd is not configured if len(cfg.Etcd.Endpoints) == 0 { etcdPod := componentPod(api.Container{ - Name: etcd, - Command: []string{ - "etcd", - "--listen-client-urls=http://127.0.0.1:2379", - "--advertise-client-urls=http://127.0.0.1:2379", - "--data-dir=/var/lib/etcd", - }, - VolumeMounts: []api.VolumeMount{certsVolumeMount(), etcdVolumeMount(), k8sVolumeMount()}, + Name: etcd, + Command: getEtcdCommand(cfg), + VolumeMounts: []api.VolumeMount{certsVolumeMount(), etcdVolumeMount(cfg.Etcd.DataDir), k8sVolumeMount()}, Image: images.GetCoreImage(images.KubeEtcdImage, cfg, kubeadmapi.GlobalEnvParams.EtcdImage), LivenessProbe: componentProbe(2379, "/health", api.URISchemeHTTP), }, certsVolume(cfg), etcdVolume(cfg), k8sVolume(cfg)) etcdPod.Spec.SecurityContext = &api.PodSecurityContext{ SELinuxOptions: &api.SELinuxOptions{ - // Unconfine the etcd container so it can write to /var/lib/etcd with SELinux enforcing: + // Unconfine the etcd container so it can write to the data dir with SELinux enforcing: Type: "spc_t", }, } @@ -146,15 +141,15 @@ func etcdVolume(cfg *kubeadmapi.MasterConfiguration) api.Volume { return api.Volume{ Name: "etcd", VolumeSource: api.VolumeSource{ - HostPath: &api.HostPathVolumeSource{Path: kubeadmapi.GlobalEnvParams.HostEtcdPath}, + HostPath: &api.HostPathVolumeSource{Path: cfg.Etcd.DataDir}, }, } } -func etcdVolumeMount() api.VolumeMount { +func etcdVolumeMount(dataDir string) api.VolumeMount { return api.VolumeMount{ Name: "etcd", - MountPath: "/var/lib/etcd", + MountPath: dataDir, } } @@ -367,6 +362,21 @@ func getAPIServerCommand(cfg *kubeadmapi.MasterConfiguration, selfHosted bool) [ return command } +func getEtcdCommand(cfg *kubeadmapi.MasterConfiguration) []string { + var command []string + + defaultArguments := map[string]string{ + "listen-client-urls": "http://127.0.0.1:2379", + "advertise-client-urls": "http://127.0.0.1:2379", + "data-dir": cfg.Etcd.DataDir, + } + + command = append(command, "etcd") + command = append(command, getExtraParameters(cfg.Etcd.ExtraArgs, defaultArguments)...) + + return command +} + func getControllerManagerCommand(cfg *kubeadmapi.MasterConfiguration, selfHosted bool) []string { var command []string diff --git a/cmd/kubeadm/app/master/manifests_test.go b/cmd/kubeadm/app/master/manifests_test.go index f9fb8ac23d1..09848ce88cd 100644 --- a/cmd/kubeadm/app/master/manifests_test.go +++ b/cmd/kubeadm/app/master/manifests_test.go @@ -31,7 +31,10 @@ import ( kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" ) -const testCertsDir = "/var/lib/certs" +const ( + testCertsDir = "/var/lib/certs" + etcdDataDir = "/var/lib/etcd" +) func TestWriteStaticPodManifests(t *testing.T) { tmpdir, err := ioutil.TempDir("", "") @@ -122,12 +125,13 @@ func TestEtcdVolume(t *testing.T) { expected api.Volume }{ { - cfg: &kubeadmapi.MasterConfiguration{}, + cfg: &kubeadmapi.MasterConfiguration{ + Etcd: kubeadmapi.Etcd{DataDir: etcdDataDir}, + }, expected: api.Volume{ Name: "etcd", VolumeSource: api.VolumeSource{ - HostPath: &api.HostPathVolumeSource{ - Path: kubeadmapi.GlobalEnvParams.HostEtcdPath}, + HostPath: &api.HostPathVolumeSource{Path: etcdDataDir}, }}, }, } @@ -158,13 +162,13 @@ func TestEtcdVolumeMount(t *testing.T) { { expected: api.VolumeMount{ Name: "etcd", - MountPath: "/var/lib/etcd", + MountPath: etcdDataDir, }, }, } for _, rt := range tests { - actual := etcdVolumeMount() + actual := etcdVolumeMount(etcdDataDir) if actual.Name != rt.expected.Name { t.Errorf( "failed etcdVolumeMount:\n\texpected: %s\n\t actual: %s", @@ -624,6 +628,62 @@ func TestGetControllerManagerCommand(t *testing.T) { } } +func TestGetEtcdCommand(t *testing.T) { + var tests = []struct { + cfg *kubeadmapi.MasterConfiguration + expected []string + }{ + { + cfg: &kubeadmapi.MasterConfiguration{ + Etcd: kubeadmapi.Etcd{DataDir: "/var/lib/etcd"}, + }, + expected: []string{ + "etcd", + "--listen-client-urls=http://127.0.0.1:2379", + "--advertise-client-urls=http://127.0.0.1:2379", + "--data-dir=/var/lib/etcd", + }, + }, + { + cfg: &kubeadmapi.MasterConfiguration{ + Etcd: kubeadmapi.Etcd{ + DataDir: "/var/lib/etcd", + ExtraArgs: map[string]string{ + "listen-client-urls": "http://10.0.1.10:2379", + "advertise-client-urls": "http://10.0.1.10:2379", + }, + }, + }, + expected: []string{ + "etcd", + "--listen-client-urls=http://10.0.1.10:2379", + "--advertise-client-urls=http://10.0.1.10:2379", + "--data-dir=/var/lib/etcd", + }, + }, + { + cfg: &kubeadmapi.MasterConfiguration{ + Etcd: kubeadmapi.Etcd{DataDir: "/etc/foo"}, + }, + expected: []string{ + "etcd", + "--listen-client-urls=http://127.0.0.1:2379", + "--advertise-client-urls=http://127.0.0.1:2379", + "--data-dir=/etc/foo", + }, + }, + } + + for _, rt := range tests { + actual := getEtcdCommand(rt.cfg) + sort.Strings(actual) + sort.Strings(rt.expected) + if !reflect.DeepEqual(actual, rt.expected) { + t.Errorf("failed getEtcdCommand:\nexpected:\n%v\nsaw:\n%v", rt.expected, actual) + } + } +} + func TestGetSchedulerCommand(t *testing.T) { var tests = []struct { cfg *kubeadmapi.MasterConfiguration diff --git a/cmd/kubeadm/app/preflight/checks.go b/cmd/kubeadm/app/preflight/checks.go index f139863889c..92c6bf7ec5b 100644 --- a/cmd/kubeadm/app/preflight/checks.go +++ b/cmd/kubeadm/app/preflight/checks.go @@ -515,7 +515,7 @@ func RunInitMasterChecks(cfg *kubeadmapi.MasterConfiguration) error { // Only do etcd related checks when no external endpoints were specified checks = append(checks, PortOpenCheck{port: 2379}, - DirAvailableCheck{Path: "/var/lib/etcd"}, + DirAvailableCheck{Path: cfg.Etcd.DataDir}, ) } else { // Only check etcd version when external endpoints are specified