diff --git a/cluster/gce/gci/BUILD b/cluster/gce/gci/BUILD index 5d138710e05..9a44af695b6 100644 --- a/cluster/gce/gci/BUILD +++ b/cluster/gce/gci/BUILD @@ -5,12 +5,14 @@ load("@io_k8s_repo_infra//defs:build.bzl", "release_filegroup") go_test( name = "go_default_test", srcs = [ - "apiserver_manifest_test.go", + "apiserver_etcd_test.go", + "apiserver_kms_test.go", "audit_policy_test.go", "configure_helper_test.go", ], data = [ ":scripts-test-data", + ":testdata", "//cluster/gce/manifests", ], deps = [ @@ -81,3 +83,8 @@ filegroup( "configure-kubeapiserver.sh", ], ) + +filegroup( + name = "testdata", + srcs = glob(["testdata/**"]), +) diff --git a/cluster/gce/gci/apiserver_etcd_test.go b/cluster/gce/gci/apiserver_etcd_test.go new file mode 100644 index 00000000000..d9fde354cb2 --- /dev/null +++ b/cluster/gce/gci/apiserver_etcd_test.go @@ -0,0 +1,95 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gci + +import ( + "strings" + "testing" +) + +type kubeAPIServeETCDEnv struct { + KubeHome string + ETCDServers string + CAKey string + CACert string + CACertPath string + APIServerKey string + APIServerCert string + APIServerCertPath string + APIServerKeyPath string + ETCDKey string + ETCDCert string +} + +func TestTLSFlags(t *testing.T) { + testCases := []struct { + desc string + env kubeAPIServeETCDEnv + want []string + }{ + { + desc: "mTLS enabled", + env: kubeAPIServeETCDEnv{ + CAKey: "CAKey", + CACert: "CACert", + CACertPath: "CACertPath", + APIServerKey: "APIServerKey", + APIServerCert: "APIServerCert", + ETCDKey: "ETCDKey", + ETCDCert: "ETCDCert", + ETCDServers: "https://127.0.0.1:2379", + APIServerKeyPath: "APIServerKeyPath", + APIServerCertPath: "APIServerCertPath", + }, + want: []string{ + "--etcd-servers=https://127.0.0.1:2379", + "--etcd-cafile=CACertPath", + "--etcd-certfile=APIServerCertPath", + "--etcd-keyfile=APIServerKeyPath", + }, + }, + { + desc: "mTLS disabled", + want: []string{"--etcd-servers=http://127.0.0.1:2379"}, + }, + } + + for _, tc := range testCases { + t.Run(tc.desc, func(t *testing.T) { + c := newManifestTestCase(t, kubeAPIServerManifestFileName, kubeAPIServerStartFuncName, nil) + defer c.tearDown() + tc.env.KubeHome = c.kubeHome + + c.mustInvokeFunc( + tc.env, + kubeAPIServerConfigScriptName, + "etcd.template", + "testdata/kube-apiserver/base.template", + "testdata/kube-apiserver/etcd.template", + ) + c.mustLoadPodFromManifest() + + execArgs := c.pod.Spec.Containers[0].Command[2] + for _, f := range tc.want { + if !strings.Contains(execArgs, f) { + t.Fatalf("Got %q, want it to contain %q", execArgs, f) + } + } + + }) + } +} diff --git a/cluster/gce/gci/apiserver_manifest_test.go b/cluster/gce/gci/apiserver_kms_test.go similarity index 67% rename from cluster/gce/gci/apiserver_manifest_test.go rename to cluster/gce/gci/apiserver_kms_test.go index d1bbe9e18e0..84f5f5020c3 100644 --- a/cluster/gce/gci/apiserver_manifest_test.go +++ b/cluster/gce/gci/apiserver_kms_test.go @@ -31,41 +31,6 @@ import ( ) const ( - /* - Template for defining the environment state of configure-helper.sh - The environment of configure-helper.sh is initially configured via kube-env file. However, as deploy-helper - executes new variables are created. ManifestTestCase does not care where a variable came from. However, future - test scenarios, may require such a distinction. - The list of variables is, by no means, complete - this is what is required to run currently defined tests. - */ - deployHelperEnv = ` -readonly KUBE_HOME={{.KubeHome}} -readonly KUBE_API_SERVER_LOG_PATH=${KUBE_HOME}/kube-apiserver.log -readonly KUBE_API_SERVER_AUDIT_LOG_PATH=${KUBE_HOME}/kube-apiserver-audit.log -readonly CLOUD_CONFIG_OPT=--cloud-config=/etc/gce.conf -readonly CA_CERT_BUNDLE_PATH=/foo/bar -readonly APISERVER_SERVER_CERT_PATH=/foo/bar -readonly APISERVER_SERVER_KEY_PATH=/foo/bar -readonly APISERVER_CLIENT_CERT_PATH=/foo/bar -readonly CLOUD_CONFIG_MOUNT="{\"name\": \"cloudconfigmount\",\"mountPath\": \"/etc/gce.conf\", \"readOnly\": true}," -readonly CLOUD_CONFIG_VOLUME="{\"name\": \"cloudconfigmount\",\"hostPath\": {\"path\": \"/etc/gce.conf\", \"type\": \"FileOrCreate\"}}," -readonly INSECURE_PORT_MAPPING="{ \"name\": \"local\", \"containerPort\": 8080, \"hostPort\": 8080}," -readonly DOCKER_REGISTRY="k8s.gcr.io" -readonly ENABLE_LEGACY_ABAC=false -readonly ETC_MANIFESTS=${KUBE_HOME}/etc/kubernetes/manifests -readonly KUBE_API_SERVER_DOCKER_TAG=v1.11.0-alpha.0.1808_3c7452dc11645d-dirty -readonly LOG_OWNER_USER=$(id -un) -readonly LOG_OWNER_GROUP=$(id -gn) -readonly SERVICEACCOUNT_ISSUER=https://foo.bar.baz -readonly SERVICEACCOUNT_KEY_PATH=/foo/bar/baz.key -{{if .EncryptionProviderConfig}} -ENCRYPTION_PROVIDER_CONFIG={{.EncryptionProviderConfig}} -{{end}} -ENCRYPTION_PROVIDER_CONFIG_PATH={{.EncryptionProviderConfigPath}} -{{if .CloudKMSIntegration}} -readonly CLOUD_KMS_INTEGRATION=true -{{end}} -` kubeAPIServerManifestFileName = "kube-apiserver.manifest" kubeAPIServerConfigScriptName = "configure-kubeapiserver.sh" kubeAPIServerStartFuncName = "start-kube-apiserver" @@ -78,21 +43,6 @@ type kubeAPIServerEnv struct { CloudKMSIntegration bool } -type kubeAPIServerManifestTestCase struct { - *ManifestTestCase -} - -func newKubeAPIServerManifestTestCase(t *testing.T) *kubeAPIServerManifestTestCase { - return &kubeAPIServerManifestTestCase{ - ManifestTestCase: newManifestTestCase(t, kubeAPIServerManifestFileName, kubeAPIServerStartFuncName, nil), - } -} - -func (c *kubeAPIServerManifestTestCase) invokeTest(e kubeAPIServerEnv, kubeEnv string) { - c.mustInvokeFunc(kubeEnv, kubeAPIServerConfigScriptName, e) - c.mustLoadPodFromManifest() -} - func TestEncryptionProviderFlag(t *testing.T) { var ( // command": [ @@ -122,7 +72,7 @@ func TestEncryptionProviderFlag(t *testing.T) { for _, tc := range testCases { t.Run(tc.desc, func(t *testing.T) { - c := newKubeAPIServerManifestTestCase(t) + c := newManifestTestCase(t, kubeAPIServerManifestFileName, kubeAPIServerStartFuncName, nil) defer c.tearDown() e := kubeAPIServerEnv{ @@ -131,7 +81,13 @@ func TestEncryptionProviderFlag(t *testing.T) { EncryptionProviderConfig: tc.encryptionProviderConfig, } - c.invokeTest(e, deployHelperEnv) + c.mustInvokeFunc( + e, + kubeAPIServerConfigScriptName, + "kms.template", + "testdata/kube-apiserver/base.template", + "testdata/kube-apiserver/kms.template") + c.mustLoadPodFromManifest() execArgs := c.pod.Spec.Containers[0].Command[execArgsIndex] flagIsInArg := strings.Contains(execArgs, encryptionConfigFlag) @@ -150,7 +106,7 @@ func TestEncryptionProviderFlag(t *testing.T) { } func TestEncryptionProviderConfig(t *testing.T) { - c := newKubeAPIServerManifestTestCase(t) + c := newManifestTestCase(t, kubeAPIServerManifestFileName, kubeAPIServerStartFuncName, nil) defer c.tearDown() p := filepath.Join(c.kubeHome, "encryption-provider-config.yaml") @@ -160,7 +116,14 @@ func TestEncryptionProviderConfig(t *testing.T) { EncryptionProviderConfig: base64.StdEncoding.EncodeToString([]byte("foo")), } - c.mustInvokeFunc(deployHelperEnv, kubeAPIServerConfigScriptName, e) + c.mustInvokeFunc( + e, + kubeAPIServerConfigScriptName, + "kms.template", + + "testdata/kube-apiserver/base.template", + "testdata/kube-apiserver/kms.template", + ) if _, err := os.Stat(p); err != nil { c.t.Fatalf("Expected encryption provider config to be written to %s, but stat failed with error: %v", p, err) @@ -214,7 +177,7 @@ func TestKMSIntegration(t *testing.T) { for _, tc := range testCases { t.Run(tc.desc, func(t *testing.T) { - c := newKubeAPIServerManifestTestCase(t) + c := newManifestTestCase(t, kubeAPIServerManifestFileName, kubeAPIServerStartFuncName, nil) defer c.tearDown() var e = kubeAPIServerEnv{ @@ -224,7 +187,15 @@ func TestKMSIntegration(t *testing.T) { CloudKMSIntegration: tc.cloudKMSIntegration, } - c.invokeTest(e, deployHelperEnv) + c.mustInvokeFunc( + e, + kubeAPIServerConfigScriptName, + "kms.template", + + "testdata/kube-apiserver/base.template", + "testdata/kube-apiserver/kms.template", + ) + c.mustLoadPodFromManifest() // By this point, we can be sure that kube-apiserver manifest is a valid POD. var gotVolume v1.Volume diff --git a/cluster/gce/gci/audit_policy_test.go b/cluster/gce/gci/audit_policy_test.go index 52efbb89241..2366cc37a3f 100644 --- a/cluster/gce/gci/audit_policy_test.go +++ b/cluster/gce/gci/audit_policy_test.go @@ -52,8 +52,12 @@ func TestCreateMasterAuditPolicy(t *testing.T) { defer c.tearDown() // Initialize required environment variables. - const kubeEnvTmpl = `readonly KUBE_HOME={{.KubeHome}}` - c.mustInvokeFunc(kubeEnvTmpl, "configure-helper.sh", kubeAPIServerEnv{KubeHome: c.kubeHome}) + c.mustInvokeFunc( + kubeAPIServerEnv{KubeHome: c.kubeHome}, + "configure-helper.sh", + "base.template", + "testdata/kube-apiserver/base.template", + ) policy, err := auditpolicy.LoadPolicyFromFile(policyFile) require.NoError(t, err, "Failed to load generated policy.") diff --git a/cluster/gce/gci/configure_helper_test.go b/cluster/gce/gci/configure_helper_test.go index ce587f6a18e..d76500a9be6 100644 --- a/cluster/gce/gci/configure_helper_test.go +++ b/cluster/gce/gci/configure_helper_test.go @@ -106,35 +106,38 @@ func (c *ManifestTestCase) mustCreateManifestDstDir() { } } -func (c *ManifestTestCase) mustCreateEnv(envTemplate string, env interface{}) string { +func (c *ManifestTestCase) mustInvokeFunc(env interface{}, scriptName, targetTemplate string, templates ...string) { + envScriptPath := c.mustCreateEnv(env, targetTemplate, templates...) + args := fmt.Sprintf("source %q ; source %q; %s", envScriptPath, scriptName, c.manifestFuncName) + cmd := exec.Command("bash", "-c", args) + + bs, err := cmd.CombinedOutput() + if err != nil { + c.t.Logf("%q", bs) + c.t.Fatalf("Failed to run %q: %v", scriptName, err) + } + c.t.Logf("%s", string(bs)) +} + +func (c *ManifestTestCase) mustCreateEnv(env interface{}, target string, templates ...string) string { f, err := os.Create(filepath.Join(c.kubeHome, envScriptFileName)) if err != nil { c.t.Fatalf("Failed to create envScript: %v", err) } defer f.Close() - t := template.Must(template.New("env").Parse(envTemplate)) + t, err := template.ParseFiles(templates...) + if err != nil { + c.t.Fatalf("Failed to parse files %q, err: %v", templates, err) + } - if err = t.Execute(f, env); err != nil { - c.t.Fatalf("Failed to execute template: %v", err) + if err = t.ExecuteTemplate(f, target, env); err != nil { + c.t.Fatalf("Failed to execute template %s, err: %v", target, err) } return f.Name() } -func (c *ManifestTestCase) mustInvokeFunc(envTemplate, scriptName string, env interface{}) { - envScriptPath := c.mustCreateEnv(envTemplate, env) - args := fmt.Sprintf("source %s ; source %s; %s", envScriptPath, scriptName, c.manifestFuncName) - cmd := exec.Command("bash", "-c", args) - - bs, err := cmd.CombinedOutput() - if err != nil { - c.t.Logf("%s", bs) - c.t.Fatalf("Failed to run %s: %v", scriptName, err) - } - c.t.Logf("%s", string(bs)) -} - func (c *ManifestTestCase) mustLoadPodFromManifest() { json, err := ioutil.ReadFile(c.manifestDestination) if err != nil { diff --git a/cluster/gce/gci/testdata/kube-apiserver/base.template b/cluster/gce/gci/testdata/kube-apiserver/base.template new file mode 100644 index 00000000000..3ec566991a7 --- /dev/null +++ b/cluster/gce/gci/testdata/kube-apiserver/base.template @@ -0,0 +1,22 @@ +{{ define "base" }} +readonly KUBE_HOME={{.}} +readonly KUBE_API_SERVER_LOG_PATH=${KUBE_HOME}/kube-apiserver.log +readonly KUBE_API_SERVER_AUDIT_LOG_PATH=${KUBE_HOME}/kube-apiserver-audit.log +readonly CLOUD_CONFIG_OPT=--cloud-config=/etc/gce.conf +readonly CA_CERT_BUNDLE_PATH=/foo/bar +readonly APISERVER_SERVER_CERT_PATH=/foo/bar +readonly APISERVER_SERVER_KEY_PATH=/foo/bar +readonly APISERVER_CLIENT_CERT_PATH=/foo/bar +readonly CLOUD_CONFIG_MOUNT="{\"name\": \"cloudconfigmount\",\"mountPath\": \"/etc/gce.conf\", \"readOnly\": true}," +readonly CLOUD_CONFIG_VOLUME="{\"name\": \"cloudconfigmount\",\"hostPath\": {\"path\": \"/etc/gce.conf\", \"type\": \"FileOrCreate\"}}," +readonly INSECURE_PORT_MAPPING="{ \"name\": \"local\", \"containerPort\": 8080, \"hostPort\": 8080}," +readonly DOCKER_REGISTRY="k8s.gcr.io" +readonly ENABLE_LEGACY_ABAC=false +readonly ETC_MANIFESTS=${KUBE_HOME}/etc/kubernetes/manifests +readonly KUBE_API_SERVER_DOCKER_TAG=v1.11.0-alpha.0.1808_3c7452dc11645d-dirty +readonly LOG_OWNER_USER=$(id -un) +readonly LOG_OWNER_GROUP=$(id -gn) +readonly SERVICEACCOUNT_ISSUER=https://foo.bar.baz +readonly SERVICEACCOUNT_KEY_PATH=/foo/bar/baz.key +{{end}}} + diff --git a/cluster/gce/gci/testdata/kube-apiserver/etcd.template b/cluster/gce/gci/testdata/kube-apiserver/etcd.template new file mode 100644 index 00000000000..ccc10e2dbb6 --- /dev/null +++ b/cluster/gce/gci/testdata/kube-apiserver/etcd.template @@ -0,0 +1,11 @@ +{{ template "base" .KubeHome }} +readonly ETCD_APISERVER_CA_KEY={{.CAKey}} +readonly ETCD_APISERVER_CA_CERT={{.CACert}} +readonly ETCD_APISERVER_SERVER_KEY={{.APIServerKey}} +readonly ETCD_APISERVER_SERVER_CERT={{.APIServerCert}} +readonly ETCD_APISERVER_CLIENT_KEY={{.ETCDKey}} +readonly ETCD_APISERVER_CLIENT_CERT={{.ETCDCert}} +readonly ETCD_SERVERS={{.ETCDServers}} +readonly ETCD_APISERVER_CA_CERT_PATH={{.CACertPath}} +readonly ETCD_APISERVER_CLIENT_CERT_PATH={{.APIServerCertPath}} +readonly ETCD_APISERVER_CLIENT_KEY_PATH={{.APIServerKeyPath}} diff --git a/cluster/gce/gci/testdata/kube-apiserver/kms.template b/cluster/gce/gci/testdata/kube-apiserver/kms.template new file mode 100644 index 00000000000..db4a14eeab7 --- /dev/null +++ b/cluster/gce/gci/testdata/kube-apiserver/kms.template @@ -0,0 +1,8 @@ +{{ template "base" .KubeHome }} +ENCRYPTION_PROVIDER_CONFIG_PATH={{.EncryptionProviderConfigPath}} +{{if .EncryptionProviderConfig}} + ENCRYPTION_PROVIDER_CONFIG={{.EncryptionProviderConfig}} +{{end}} +{{if .CloudKMSIntegration}} + readonly CLOUD_KMS_INTEGRATION=true +{{end}} \ No newline at end of file