Merge pull request #84579 from immutableT/kas-etcd-params

Refactor tests for configure-helper.sh by moving environment config to testdata.

cluster/gce/gci/BUILD

@@ -5,12 +5,14 @@ load("@io_k8s_repo_infra//defs:build.bzl", "release_filegroup")
 go_test(
     name = "go_default_test",
     srcs = [
-        "apiserver_manifest_test.go",
+        "apiserver_etcd_test.go",
+        "apiserver_kms_test.go",
         "audit_policy_test.go",
         "configure_helper_test.go",
     ],
     data = [
         ":scripts-test-data",
+        ":testdata",
         "//cluster/gce/manifests",
     ],
     deps = [
@@ -81,3 +83,8 @@ filegroup(
         "configure-kubeapiserver.sh",
     ],
 )
+
+filegroup(
+    name = "testdata",
+    srcs = glob(["testdata/**"]),
+)

cluster/gce/gci/apiserver_etcd_test.go

@@ -0,0 +1,95 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package gci
+
+import (
+	"strings"
+	"testing"
+)
+
+type kubeAPIServeETCDEnv struct {
+	KubeHome          string
+	ETCDServers       string
+	CAKey             string
+	CACert            string
+	CACertPath        string
+	APIServerKey      string
+	APIServerCert     string
+	APIServerCertPath string
+	APIServerKeyPath  string
+	ETCDKey           string
+	ETCDCert          string
+}
+
+func TestTLSFlags(t *testing.T) {
+	testCases := []struct {
+		desc string
+		env  kubeAPIServeETCDEnv
+		want []string
+	}{
+		{
+			desc: "mTLS enabled",
+			env: kubeAPIServeETCDEnv{
+				CAKey:             "CAKey",
+				CACert:            "CACert",
+				CACertPath:        "CACertPath",
+				APIServerKey:      "APIServerKey",
+				APIServerCert:     "APIServerCert",
+				ETCDKey:           "ETCDKey",
+				ETCDCert:          "ETCDCert",
+				ETCDServers:       "https://127.0.0.1:2379",
+				APIServerKeyPath:  "APIServerKeyPath",
+				APIServerCertPath: "APIServerCertPath",
+			},
+			want: []string{
+				"--etcd-servers=https://127.0.0.1:2379",
+				"--etcd-cafile=CACertPath",
+				"--etcd-certfile=APIServerCertPath",
+				"--etcd-keyfile=APIServerKeyPath",
+			},
+		},
+		{
+			desc: "mTLS disabled",
+			want: []string{"--etcd-servers=http://127.0.0.1:2379"},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.desc, func(t *testing.T) {
+			c := newManifestTestCase(t, kubeAPIServerManifestFileName, kubeAPIServerStartFuncName, nil)
+			defer c.tearDown()
+			tc.env.KubeHome = c.kubeHome
+
+			c.mustInvokeFunc(
+				tc.env,
+				kubeAPIServerConfigScriptName,
+				"etcd.template",
+				"testdata/kube-apiserver/base.template",
+				"testdata/kube-apiserver/etcd.template",
+			)
+			c.mustLoadPodFromManifest()
+
+			execArgs := c.pod.Spec.Containers[0].Command[2]
+			for _, f := range tc.want {
+				if !strings.Contains(execArgs, f) {
+					t.Fatalf("Got %q, want it to contain %q", execArgs, f)
+				}
+			}
+
+		})
+	}
+}

cluster/gce/gci/apiserver_kms_test.go

@@ -31,41 +31,6 @@ import (
 )
 
 const (
-	/*
-	   Template for defining the environment state of configure-helper.sh
-	   The environment of configure-helper.sh is initially configured via kube-env file. However, as deploy-helper
-	   executes new variables are created. ManifestTestCase does not care where a variable came from. However, future
-	   test scenarios, may require such a distinction.
-	   The list of variables is, by no means, complete - this is what is required to run currently defined tests.
-	*/
-	deployHelperEnv = `
-readonly KUBE_HOME={{.KubeHome}}
-readonly KUBE_API_SERVER_LOG_PATH=${KUBE_HOME}/kube-apiserver.log
-readonly KUBE_API_SERVER_AUDIT_LOG_PATH=${KUBE_HOME}/kube-apiserver-audit.log
-readonly CLOUD_CONFIG_OPT=--cloud-config=/etc/gce.conf
-readonly CA_CERT_BUNDLE_PATH=/foo/bar
-readonly APISERVER_SERVER_CERT_PATH=/foo/bar
-readonly APISERVER_SERVER_KEY_PATH=/foo/bar
-readonly APISERVER_CLIENT_CERT_PATH=/foo/bar
-readonly CLOUD_CONFIG_MOUNT="{\"name\": \"cloudconfigmount\",\"mountPath\": \"/etc/gce.conf\", \"readOnly\": true},"
-readonly CLOUD_CONFIG_VOLUME="{\"name\": \"cloudconfigmount\",\"hostPath\": {\"path\": \"/etc/gce.conf\", \"type\": \"FileOrCreate\"}},"
-readonly INSECURE_PORT_MAPPING="{ \"name\": \"local\", \"containerPort\": 8080, \"hostPort\": 8080},"
-readonly DOCKER_REGISTRY="k8s.gcr.io"
-readonly ENABLE_LEGACY_ABAC=false
-readonly ETC_MANIFESTS=${KUBE_HOME}/etc/kubernetes/manifests
-readonly KUBE_API_SERVER_DOCKER_TAG=v1.11.0-alpha.0.1808_3c7452dc11645d-dirty
-readonly LOG_OWNER_USER=$(id -un)
-readonly LOG_OWNER_GROUP=$(id -gn)
-readonly SERVICEACCOUNT_ISSUER=https://foo.bar.baz
-readonly SERVICEACCOUNT_KEY_PATH=/foo/bar/baz.key
-{{if .EncryptionProviderConfig}}
-ENCRYPTION_PROVIDER_CONFIG={{.EncryptionProviderConfig}}
-{{end}}
-ENCRYPTION_PROVIDER_CONFIG_PATH={{.EncryptionProviderConfigPath}}
-{{if .CloudKMSIntegration}}
-readonly CLOUD_KMS_INTEGRATION=true
-{{end}}
-`
 	kubeAPIServerManifestFileName = "kube-apiserver.manifest"
 	kubeAPIServerConfigScriptName = "configure-kubeapiserver.sh"
 	kubeAPIServerStartFuncName    = "start-kube-apiserver"
@@ -78,21 +43,6 @@ type kubeAPIServerEnv struct {
 	CloudKMSIntegration          bool
 }
 
-type kubeAPIServerManifestTestCase struct {
-	*ManifestTestCase
-}
-
-func newKubeAPIServerManifestTestCase(t *testing.T) *kubeAPIServerManifestTestCase {
-	return &kubeAPIServerManifestTestCase{
-		ManifestTestCase: newManifestTestCase(t, kubeAPIServerManifestFileName, kubeAPIServerStartFuncName, nil),
-	}
-}
-
-func (c *kubeAPIServerManifestTestCase) invokeTest(e kubeAPIServerEnv, kubeEnv string) {
-	c.mustInvokeFunc(kubeEnv, kubeAPIServerConfigScriptName, e)
-	c.mustLoadPodFromManifest()
-}
-
 func TestEncryptionProviderFlag(t *testing.T) {
 	var (
 		//	command": [
@@ -122,7 +72,7 @@ func TestEncryptionProviderFlag(t *testing.T) {
 
 	for _, tc := range testCases {
 		t.Run(tc.desc, func(t *testing.T) {
-			c := newKubeAPIServerManifestTestCase(t)
+			c := newManifestTestCase(t, kubeAPIServerManifestFileName, kubeAPIServerStartFuncName, nil)
 			defer c.tearDown()
 
 			e := kubeAPIServerEnv{
@@ -131,7 +81,13 @@ func TestEncryptionProviderFlag(t *testing.T) {
 				EncryptionProviderConfig:     tc.encryptionProviderConfig,
 			}
 
-			c.invokeTest(e, deployHelperEnv)
+			c.mustInvokeFunc(
+				e,
+				kubeAPIServerConfigScriptName,
+				"kms.template",
+				"testdata/kube-apiserver/base.template",
+				"testdata/kube-apiserver/kms.template")
+			c.mustLoadPodFromManifest()
 
 			execArgs := c.pod.Spec.Containers[0].Command[execArgsIndex]
 			flagIsInArg := strings.Contains(execArgs, encryptionConfigFlag)
@@ -150,7 +106,7 @@ func TestEncryptionProviderFlag(t *testing.T) {
 }
 
 func TestEncryptionProviderConfig(t *testing.T) {
-	c := newKubeAPIServerManifestTestCase(t)
+	c := newManifestTestCase(t, kubeAPIServerManifestFileName, kubeAPIServerStartFuncName, nil)
 	defer c.tearDown()
 
 	p := filepath.Join(c.kubeHome, "encryption-provider-config.yaml")
@@ -160,7 +116,14 @@ func TestEncryptionProviderConfig(t *testing.T) {
 		EncryptionProviderConfig:     base64.StdEncoding.EncodeToString([]byte("foo")),
 	}
 
-	c.mustInvokeFunc(deployHelperEnv, kubeAPIServerConfigScriptName, e)
+	c.mustInvokeFunc(
+		e,
+		kubeAPIServerConfigScriptName,
+		"kms.template",
+
+		"testdata/kube-apiserver/base.template",
+		"testdata/kube-apiserver/kms.template",
+	)
 
 	if _, err := os.Stat(p); err != nil {
 		c.t.Fatalf("Expected encryption provider config to be written to %s, but stat failed with error: %v", p, err)
@@ -214,7 +177,7 @@ func TestKMSIntegration(t *testing.T) {
 
 	for _, tc := range testCases {
 		t.Run(tc.desc, func(t *testing.T) {
-			c := newKubeAPIServerManifestTestCase(t)
+			c := newManifestTestCase(t, kubeAPIServerManifestFileName, kubeAPIServerStartFuncName, nil)
 			defer c.tearDown()
 
 			var e = kubeAPIServerEnv{
@@ -224,7 +187,15 @@ func TestKMSIntegration(t *testing.T) {
 				CloudKMSIntegration:          tc.cloudKMSIntegration,
 			}
 
-			c.invokeTest(e, deployHelperEnv)
+			c.mustInvokeFunc(
+				e,
+				kubeAPIServerConfigScriptName,
+				"kms.template",
+
+				"testdata/kube-apiserver/base.template",
+				"testdata/kube-apiserver/kms.template",
+			)
+			c.mustLoadPodFromManifest()
 			// By this point, we can be sure that kube-apiserver manifest is a valid POD.
 
 			var gotVolume v1.Volume

cluster/gce/gci/audit_policy_test.go

@@ -52,8 +52,12 @@ func TestCreateMasterAuditPolicy(t *testing.T) {
 	defer c.tearDown()
 
 	// Initialize required environment variables.
-	const kubeEnvTmpl = `readonly KUBE_HOME={{.KubeHome}}`
+	c.mustInvokeFunc(
-	c.mustInvokeFunc(kubeEnvTmpl, "configure-helper.sh", kubeAPIServerEnv{KubeHome: c.kubeHome})
+		kubeAPIServerEnv{KubeHome: c.kubeHome},
+		"configure-helper.sh",
+		"base.template",
+		"testdata/kube-apiserver/base.template",
+	)
 
 	policy, err := auditpolicy.LoadPolicyFromFile(policyFile)
 	require.NoError(t, err, "Failed to load generated policy.")

cluster/gce/gci/configure_helper_test.go

@@ -106,35 +106,38 @@ func (c *ManifestTestCase) mustCreateManifestDstDir() {
 	}
 }
 
-func (c *ManifestTestCase) mustCreateEnv(envTemplate string, env interface{}) string {
+func (c *ManifestTestCase) mustInvokeFunc(env interface{}, scriptName, targetTemplate string, templates ...string) {
+	envScriptPath := c.mustCreateEnv(env, targetTemplate, templates...)
+	args := fmt.Sprintf("source %q ; source %q; %s", envScriptPath, scriptName, c.manifestFuncName)
+	cmd := exec.Command("bash", "-c", args)
+
+	bs, err := cmd.CombinedOutput()
+	if err != nil {
+		c.t.Logf("%q", bs)
+		c.t.Fatalf("Failed to run %q: %v", scriptName, err)
+	}
+	c.t.Logf("%s", string(bs))
+}
+
+func (c *ManifestTestCase) mustCreateEnv(env interface{}, target string, templates ...string) string {
 	f, err := os.Create(filepath.Join(c.kubeHome, envScriptFileName))
 	if err != nil {
 		c.t.Fatalf("Failed to create envScript: %v", err)
 	}
 	defer f.Close()
 
-	t := template.Must(template.New("env").Parse(envTemplate))
+	t, err := template.ParseFiles(templates...)
+	if err != nil {
+		c.t.Fatalf("Failed to parse files %q, err: %v", templates, err)
+	}
 
-	if err = t.Execute(f, env); err != nil {
+	if err = t.ExecuteTemplate(f, target, env); err != nil {
-		c.t.Fatalf("Failed to execute template: %v", err)
+		c.t.Fatalf("Failed to execute template %s, err: %v", target, err)
 	}
 
 	return f.Name()
 }
 
-func (c *ManifestTestCase) mustInvokeFunc(envTemplate, scriptName string, env interface{}) {
-	envScriptPath := c.mustCreateEnv(envTemplate, env)
-	args := fmt.Sprintf("source %s ; source %s; %s", envScriptPath, scriptName, c.manifestFuncName)
-	cmd := exec.Command("bash", "-c", args)
-
-	bs, err := cmd.CombinedOutput()
-	if err != nil {
-		c.t.Logf("%s", bs)
-		c.t.Fatalf("Failed to run %s: %v", scriptName, err)
-	}
-	c.t.Logf("%s", string(bs))
-}
-
 func (c *ManifestTestCase) mustLoadPodFromManifest() {
 	json, err := ioutil.ReadFile(c.manifestDestination)
 	if err != nil {

cluster/gce/gci/testdata/kube-apiserver/base.template

@@ -0,0 +1,22 @@
+{{ define "base" }}
+readonly KUBE_HOME={{.}}
+readonly KUBE_API_SERVER_LOG_PATH=${KUBE_HOME}/kube-apiserver.log
+readonly KUBE_API_SERVER_AUDIT_LOG_PATH=${KUBE_HOME}/kube-apiserver-audit.log
+readonly CLOUD_CONFIG_OPT=--cloud-config=/etc/gce.conf
+readonly CA_CERT_BUNDLE_PATH=/foo/bar
+readonly APISERVER_SERVER_CERT_PATH=/foo/bar
+readonly APISERVER_SERVER_KEY_PATH=/foo/bar
+readonly APISERVER_CLIENT_CERT_PATH=/foo/bar
+readonly CLOUD_CONFIG_MOUNT="{\"name\": \"cloudconfigmount\",\"mountPath\": \"/etc/gce.conf\", \"readOnly\": true},"
+readonly CLOUD_CONFIG_VOLUME="{\"name\": \"cloudconfigmount\",\"hostPath\": {\"path\": \"/etc/gce.conf\", \"type\": \"FileOrCreate\"}},"
+readonly INSECURE_PORT_MAPPING="{ \"name\": \"local\", \"containerPort\": 8080, \"hostPort\": 8080},"
+readonly DOCKER_REGISTRY="k8s.gcr.io"
+readonly ENABLE_LEGACY_ABAC=false
+readonly ETC_MANIFESTS=${KUBE_HOME}/etc/kubernetes/manifests
+readonly KUBE_API_SERVER_DOCKER_TAG=v1.11.0-alpha.0.1808_3c7452dc11645d-dirty
+readonly LOG_OWNER_USER=$(id -un)
+readonly LOG_OWNER_GROUP=$(id -gn)
+readonly SERVICEACCOUNT_ISSUER=https://foo.bar.baz
+readonly SERVICEACCOUNT_KEY_PATH=/foo/bar/baz.key
+{{end}}}
+

cluster/gce/gci/testdata/kube-apiserver/etcd.template

@@ -0,0 +1,11 @@
+{{ template "base" .KubeHome }}
+readonly ETCD_APISERVER_CA_KEY={{.CAKey}}
+readonly ETCD_APISERVER_CA_CERT={{.CACert}}
+readonly ETCD_APISERVER_SERVER_KEY={{.APIServerKey}}
+readonly ETCD_APISERVER_SERVER_CERT={{.APIServerCert}}
+readonly ETCD_APISERVER_CLIENT_KEY={{.ETCDKey}}
+readonly ETCD_APISERVER_CLIENT_CERT={{.ETCDCert}}
+readonly ETCD_SERVERS={{.ETCDServers}}
+readonly ETCD_APISERVER_CA_CERT_PATH={{.CACertPath}}
+readonly ETCD_APISERVER_CLIENT_CERT_PATH={{.APIServerCertPath}}
+readonly ETCD_APISERVER_CLIENT_KEY_PATH={{.APIServerKeyPath}}

cluster/gce/gci/testdata/kube-apiserver/kms.template

@@ -0,0 +1,8 @@
+{{ template "base" .KubeHome }}
+ENCRYPTION_PROVIDER_CONFIG_PATH={{.EncryptionProviderConfigPath}}
+{{if .EncryptionProviderConfig}}
+  ENCRYPTION_PROVIDER_CONFIG={{.EncryptionProviderConfig}}
+{{end}}
+{{if .CloudKMSIntegration}}
+  readonly CLOUD_KMS_INTEGRATION=true
+{{end}}