Merge pull request #62669 from immutableT/deploy_helper_test

Automatic merge from submit-queue (batch tested with PRs 63007, 62919, 62669, 62860). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Add unit test for configure-helper.sh.

**What this PR does / why we need it**:
Add a framework for unit-testing configure-helper.sh.
configure-helper.sh plays a critical role in initializing clusters both on GCE and GKE. It is currently, over 2K lines of code, yet it has no unit test coverage.
This PR proposes a framework/approach on how to provide test coverage for this component.
Notes: 
1. Changes to configure-helper.sh itself were necessary to enable sourcing of this script for the purposes of testing.
2. As POC api_manifest_test.go covers the logic related to the initialization of apiserver when integration with KMS was requested. The hope is that the same approach could be extended to the rest of the script.

**Which issue(s) this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close the issue(s) when PR gets merged)*:
Fixes #

**Special notes for your reviewer**:

**Release note**:

```release-note
NONE
```

build/root/BUILD.root

@@ -28,13 +28,13 @@ gcs_upload(
     data = [
         ":_binary-artifacts-and-hashes",
         "//build/release-tars:release-tars-and-hashes",
-        "//cluster/gce:gcs-release-artifacts-and-hashes",
+        "//cluster/gce/gci:gcs-release-artifacts-and-hashes",
     ],
     tags = ["manual"],
     upload_paths = {
         "//:_binary-artifacts-and-hashes": "bin/linux/amd64",
         "//build/release-tars:release-tars-and-hashes": "",
-        "//cluster/gce:gcs-release-artifacts-and-hashes": "extra/gce",
+        "//cluster/gce/gci:gcs-release-artifacts-and-hashes": "extra/gce",
     },
 )
 

cluster/BUILD

@@ -29,9 +29,9 @@ pkg_tar(
     package_dir = "kubernetes/gci-trusty",
     deps = [
         "//cluster/addons",
-        "//cluster/gce:gce-master-manifests",
-        "//cluster/gce:gci-trusty-manifests",
         "//cluster/gce/addons",
+        "//cluster/gce/gci:gci-trusty-manifests",
+        "//cluster/gce/manifests:gce-master-manifests",
     ],
 )
 

cluster/gce/BUILD

@@ -3,17 +3,6 @@ package(default_visibility = ["//visibility:public"])
 load("@io_kubernetes_build//defs:build.bzl", "release_filegroup")
 load("@io_kubernetes_build//defs:pkg.bzl", "pkg_tar")
 
-pkg_tar(
-    name = "gci-trusty-manifests",
-    files = {
-        "//cluster/gce/gci/mounter": "gci-mounter",
-        "gci/configure-helper.sh": "gci-configure-helper.sh",
-        "gci/health-monitor.sh": "health-monitor.sh",
-    },
-    mode = "0755",
-    strip_prefix = ".",
-)
-
 filegroup(
     name = "package-srcs",
     srcs = glob(["**"]),
@@ -26,38 +15,8 @@ filegroup(
     srcs = [
         ":package-srcs",
         "//cluster/gce/addons:all-srcs",
-        "//cluster/gce/gci/mounter:all-srcs",
+        "//cluster/gce/gci:all-srcs",
+        "//cluster/gce/manifests:all-srcs",
     ],
     tags = ["automanaged"],
 )
-
-# Having the COS code from the GCE cluster deploy hosted with the release is
-# useful for GKE.  This list should match the list in
-# kubernetes/release/lib/releaselib.sh.
-release_filegroup(
-    name = "gcs-release-artifacts",
-    srcs = [
-        "gci/configure.sh",
-        "gci/master.yaml",
-        "gci/node.yaml",
-    ],
-)
-
-pkg_tar(
-    name = "gce-master-manifests",
-    srcs = [
-        "manifests/abac-authz-policy.jsonl",
-        "manifests/cluster-autoscaler.manifest",
-        "manifests/e2e-image-puller.manifest",
-        "manifests/etcd.manifest",
-        "manifests/glbc.manifest",
-        "manifests/kms-plugin-container.manifest",
-        "manifests/kube-addon-manager.yaml",
-        "manifests/kube-apiserver.manifest",
-        "manifests/kube-controller-manager.manifest",
-        "manifests/kube-proxy.manifest",
-        "manifests/kube-scheduler.manifest",
-        "manifests/rescheduler.manifest",
-    ],
-    mode = "0644",
-)

cluster/gce/gci/BUILD

@@ -0,0 +1,69 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_test")
+load("@io_kubernetes_build//defs:pkg.bzl", "pkg_tar")
+load("@io_kubernetes_build//defs:build.bzl", "release_filegroup")
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "apiserver_manifest_test.go",
+        "configure_helper_test.go",
+    ],
+    data = [
+        ":scripts-test-data",
+        "//cluster/gce/manifests:manifests-test-data",
+    ],
+    deps = [
+        "//pkg/api/legacyscheme:go_default_library",
+        "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
+    ],
+)
+
+# Having the COS code from the GCE cluster deploy hosted with the release is
+# useful for GKE.  This list should match the list in
+# kubernetes/release/lib/releaselib.sh.
+release_filegroup(
+    name = "gcs-release-artifacts",
+    srcs = [
+        "configure.sh",
+        "master.yaml",
+        "node.yaml",
+    ],
+    visibility = ["//visibility:public"],
+)
+
+pkg_tar(
+    name = "gci-trusty-manifests",
+    files = {
+        "//cluster/gce/gci/mounter": "gci-mounter",
+        "configure-helper.sh": "gci-configure-helper.sh",
+        "health-monitor.sh": "health-monitor.sh",
+    },
+    mode = "0755",
+    strip_prefix = ".",
+    visibility = ["//visibility:public"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//cluster/gce/gci/mounter:all-srcs",
+    ],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)
+
+filegroup(
+    name = "scripts-test-data",
+    srcs = [
+        "configure-helper.sh",
+    ],
+)

cluster/gce/gci/apiserver_manifest_test.go

@@ -0,0 +1,212 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package gci
+
+import (
+	"encoding/base64"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"k8s.io/api/core/v1"
+)
+
+const (
+	/*
+	   Template for defining the environment state of configure-helper.sh
+	   The environment of configure-helper.sh is initially configured via kube-env file. However, as deploy-helper
+	   executes new variables are created. ManifestTestCase does not care where a variable came from. However, future
+	   test scenarios, may require such a distinction.
+	   The list of variables is, by no means, complete - this is what is required to run currently defined tests.
+	*/
+	deployHelperEnv = `
+readonly KUBE_HOME={{.KubeHome}}
+readonly KUBE_API_SERVER_LOG_PATH=${KUBE_HOME}/kube-apiserver.log
+readonly KUBE_API_SERVER_AUDIT_LOG_PATH=${KUBE_HOME}/kube-apiserver-audit.log
+readonly CLOUD_CONFIG_OPT=--cloud-config=/etc/gce.conf
+readonly CA_CERT_BUNDLE_PATH=/foo/bar
+readonly APISERVER_SERVER_CERT_PATH=/foo/bar
+readonly APISERVER_SERVER_KEY_PATH=/foo/bar
+readonly APISERVER_CLIENT_CERT_PATH=/foo/bar
+readonly CLOUD_CONFIG_MOUNT="{\"name\": \"cloudconfigmount\",\"mountPath\": \"/etc/gce.conf\", \"readOnly\": true},"
+readonly CLOUD_CONFIG_VOLUME="{\"name\": \"cloudconfigmount\",\"hostPath\": {\"path\": \"/etc/gce.conf\", \"type\": \"FileOrCreate\"}},"
+readonly DOCKER_REGISTRY="k8s.gcr.io"
+readonly ENABLE_LEGACY_ABAC=false
+readonly ETC_MANIFESTS=${KUBE_HOME}/etc/kubernetes/manifests
+readonly KUBE_API_SERVER_DOCKER_TAG=v1.11.0-alpha.0.1808_3c7452dc11645d-dirty
+readonly LOG_OWNER_USER=$(whoami)
+readonly LOG_OWNER_GROUP=$(id -gn)
+ENCRYPTION_PROVIDER_CONFIG={{.EncryptionProviderConfig}}
+ENCRYPTION_PROVIDER_CONFIG_PATH={{.EncryptionProviderConfigPath}}
+readonly ETCD_KMS_KEY_ID={{.ETCDKMSKeyID}}
+`
+	kubeAPIServerManifestFileName = "kube-apiserver.manifest"
+	kmsPluginManifestFileName     = "kms-plugin-container.manifest"
+	kubeAPIServerStartFuncName    = "start-kube-apiserver"
+
+	// Position of containers within a pod manifest
+	kmsPluginContainerIndex        = 0
+	apiServerContainerIndexNoKMS   = 0
+	apiServerContainerIndexWithKMS = 1
+
+	//	command": [
+	//   "/bin/sh", - Index 0
+	//   "-c",      - Index 1
+	//   "exec /usr/local/bin/kube-apiserver " - Index 2
+	execArgsIndex = 2
+
+	socketVolumeMountIndexKMSPlugin = 1
+	socketVolumeMountIndexAPIServer = 0
+)
+
+type kubeAPIServerEnv struct {
+	KubeHome                     string
+	EncryptionProviderConfig     string
+	EncryptionProviderConfigPath string
+	ETCDKMSKeyID                 string
+}
+
+type kubeAPIServerManifestTestCase struct {
+	*ManifestTestCase
+	apiServerContainer v1.Container
+	kmsPluginContainer v1.Container
+}
+
+func newKubeAPIServerManifestTestCase(t *testing.T) *kubeAPIServerManifestTestCase {
+	return &kubeAPIServerManifestTestCase{
+		ManifestTestCase: newManifestTestCase(t, kubeAPIServerManifestFileName, kubeAPIServerStartFuncName, []string{kmsPluginManifestFileName}),
+	}
+}
+
+func (c *kubeAPIServerManifestTestCase) mustLoadContainers() {
+	c.mustLoadPodFromManifest()
+
+	switch len(c.pod.Spec.Containers) {
+	case 1:
+		c.apiServerContainer = c.pod.Spec.Containers[apiServerContainerIndexNoKMS]
+	case 2:
+		c.apiServerContainer = c.pod.Spec.Containers[apiServerContainerIndexWithKMS]
+		c.kmsPluginContainer = c.pod.Spec.Containers[kmsPluginContainerIndex]
+	default:
+		c.t.Fatalf("got %d containers in apiserver pod, want 1 or 2", len(c.pod.Spec.Containers))
+	}
+}
+
+func (c *kubeAPIServerManifestTestCase) invokeTest(e kubeAPIServerEnv) {
+	c.mustInvokeFunc(deployHelperEnv, e)
+	c.mustLoadContainers()
+}
+
+func getEncryptionProviderConfigFlag(path string) string {
+	return fmt.Sprintf("--experimental-encryption-provider-config=%s", path)
+}
+
+func TestEncryptionProviderFlag(t *testing.T) {
+	c := newKubeAPIServerManifestTestCase(t)
+	defer c.tearDown()
+
+	e := kubeAPIServerEnv{
+		KubeHome:                     c.kubeHome,
+		EncryptionProviderConfig:     base64.StdEncoding.EncodeToString([]byte("FOO")),
+		EncryptionProviderConfigPath: filepath.Join(c.kubeHome, "encryption-provider-config.yaml"),
+	}
+
+	c.invokeTest(e)
+
+	expectedFlag := getEncryptionProviderConfigFlag(e.EncryptionProviderConfigPath)
+	execArgs := c.apiServerContainer.Command[execArgsIndex]
+	if !strings.Contains(execArgs, expectedFlag) {
+		c.t.Fatalf("Got %q, wanted the flag to contain %q", execArgs, expectedFlag)
+	}
+}
+
+func TestEncryptionProviderConfig(t *testing.T) {
+	c := newKubeAPIServerManifestTestCase(t)
+	defer c.tearDown()
+
+	p := filepath.Join(c.kubeHome, "encryption-provider-config.yaml")
+	e := kubeAPIServerEnv{
+		KubeHome:                     c.kubeHome,
+		EncryptionProviderConfig:     base64.StdEncoding.EncodeToString([]byte("FOO")),
+		EncryptionProviderConfigPath: p,
+	}
+
+	c.mustInvokeFunc(deployHelperEnv, e)
+
+	if _, err := os.Stat(p); err != nil {
+		c.t.Fatalf("Expected encryption provider config to be written to %s, but stat failed with error: %v", p, err)
+	}
+}
+
+// TestKMSEncryptionProviderConfig asserts that if ETCD_KMS_KEY_ID is set then start-kube-apiserver will produce
+// EncryptionProviderConfig file of type KMS and inject experimental-encryption-provider-config startup flag.
+func TestKMSEncryptionProviderConfig(t *testing.T) {
+	c := newKubeAPIServerManifestTestCase(t)
+	defer c.tearDown()
+
+	e := kubeAPIServerEnv{
+		KubeHome:                     c.kubeHome,
+		EncryptionProviderConfigPath: filepath.Join(c.kubeHome, "encryption-provider-config.yaml"),
+		ETCDKMSKeyID:                 "FOO",
+	}
+
+	c.invokeTest(e)
+
+	expectedFlag := getEncryptionProviderConfigFlag(e.EncryptionProviderConfigPath)
+	execArgs := c.apiServerContainer.Command[execArgsIndex]
+	if !strings.Contains(execArgs, expectedFlag) {
+		c.t.Fatalf("Got %q, wanted the flag to contain %q", execArgs, expectedFlag)
+	}
+
+	p := filepath.Join(c.kubeHome, "encryption-provider-config.yaml")
+	if _, err := os.Stat(p); err != nil {
+		c.t.Fatalf("Expected encryption provider config to be written to %s, but stat failed with error: %v", p, err)
+	}
+
+	d, err := ioutil.ReadFile(p)
+	if err != nil {
+		c.t.Fatalf("Failed to read encryption provider config %s", p)
+	}
+
+	if !strings.Contains(string(d), "name: grpc-kms-provider") {
+		c.t.Fatalf("Got %s\n, wanted encryption provider config to be of type grpc-kms", string(d))
+	}
+}
+
+func TestKMSPluginAndAPIServerSharedVolume(t *testing.T) {
+	c := newKubeAPIServerManifestTestCase(t)
+	defer c.tearDown()
+
+	var e = kubeAPIServerEnv{
+		KubeHome:                     c.kubeHome,
+		EncryptionProviderConfigPath: filepath.Join(c.kubeHome, "encryption-provider-config.yaml"),
+		ETCDKMSKeyID:                 "FOO",
+	}
+
+	c.invokeTest(e)
+
+	k := c.kmsPluginContainer.VolumeMounts[socketVolumeMountIndexKMSPlugin].MountPath
+	a := c.apiServerContainer.VolumeMounts[socketVolumeMountIndexAPIServer].MountPath
+
+	if k != a {
+		t.Fatalf("Got %s!=%s, wanted KMSPlugin VolumeMount #1:%s to be equal to kube-apiserver VolumeMount #0:%s",
+			k, a, k, a)
+	}
+}

cluster/gce/gci/configure-helper.sh

@@ -1179,7 +1179,7 @@ EOF
 function prepare-log-file {
   touch $1
   chmod 644 $1
-  chown root:root $1
+  chown "${LOG_OWNER_USER:-root}":"${LOG_OWNER_GROUP:-root}" $1
 }
 
 # Prepares parameters for kube-proxy manifest.
@@ -1404,8 +1404,8 @@ function prepare-mounter-rootfs {
 #   DOCKER_REGISTRY
 function start-kube-apiserver {
   echo "Start kubernetes api-server"
-  prepare-log-file /var/log/kube-apiserver.log
+  prepare-log-file "${KUBE_API_SERVER_LOG_PATH:-/var/log/kube-apiserver.log}"
-  prepare-log-file /var/log/kube-apiserver-audit.log
+  prepare-log-file "${KUBE_API_SERVER_AUDIT_LOG_PATH:-/var/log/kube-apiserver-audit.log}"
 
   # Calculate variables and assemble the command line.
   local params="${API_SERVER_TEST_LOG_LEVEL:-"--v=2"} ${APISERVER_TEST_ARGS:-} ${CLOUD_CONFIG_OPT}"
@@ -1702,14 +1702,14 @@ EOM
   fi
 
   if [[ -n "${ENCRYPTION_PROVIDER_CONFIG:-}" ]]; then
-    local encryption_provider_config_path="/etc/srv/kubernetes/encryption-provider-config.yml"
+    ENCRYPTION_PROVIDER_CONFIG_PATH="${ENCRYPTION_PROVIDER_CONFIG_PATH:-/etc/srv/kubernetes/encryption-provider-config.yml}"
-    echo "${ENCRYPTION_PROVIDER_CONFIG}" | base64 --decode > "${encryption_provider_config_path}"
+    echo "${ENCRYPTION_PROVIDER_CONFIG}" | base64 --decode > "${ENCRYPTION_PROVIDER_CONFIG_PATH}"
-    params+=" --experimental-encryption-provider-config=${encryption_provider_config_path}"
+    params+=" --experimental-encryption-provider-config=${ENCRYPTION_PROVIDER_CONFIG_PATH}"
   fi
 
   src_file="${src_dir}/kube-apiserver.manifest"
   # Evaluate variables.
-  local -r kube_apiserver_docker_tag=$(cat /home/kubernetes/kube-docker-files/kube-apiserver.docker_tag)
+  local -r kube_apiserver_docker_tag="${KUBE_API_SERVER_DOCKER_TAG:-$(cat /home/kubernetes/kube-docker-files/kube-apiserver.docker_tag)}"
   sed -i -e "s@{{params}}@${params}@g" "${src_file}"
   sed -i -e "s@{{container_env}}@${container_env}@g" ${src_file}
   sed -i -e "s@{{srv_kube_path}}@/etc/srv/kubernetes@g" "${src_file}"
@@ -1756,8 +1756,8 @@ EOM
       exit 1
     fi
 
-    if [[ ! -f "${encryption_provider_config_path}" ]]; then
+    if [[ ! -f "${ENCRYPTION_PROVIDER_CONFIG_PATH}" ]]; then
-      echo "Error: KMS Integration was requested, but "${encryption_provider_config_path}" is missing."
+      echo "Error: KMS Integration was requested, but "${ENCRYPTION_PROVIDER_CONFIG_PATH}" is missing."
       exit 1
     fi
 
@@ -1771,8 +1771,8 @@ EOM
     local kms_socket_vol="{ \"name\": \"kmssocket\", \"hostPath\": {\"path\": \"${kms_socket_dir}\", \"type\": \"DirectoryOrCreate\"}}"
     local kms_path_to_socket="${kms_socket_dir}/socket.sock"
 
-    local encryption_provider_mnt="{ \"name\": \"encryptionconfig\", \"mountPath\": \"${encryption_provider_config_path}\", \"readOnly\": true}"
+    local encryption_provider_mnt="{ \"name\": \"encryptionconfig\", \"mountPath\": \"${ENCRYPTION_PROVIDER_CONFIG_PATH}\", \"readOnly\": true}"
-    local encryption_provider_vol="{ \"name\": \"encryptionconfig\", \"hostPath\": {\"path\": \"${encryption_provider_config_path}\", \"type\": \"File\"}}"
+    local encryption_provider_vol="{ \"name\": \"encryptionconfig\", \"hostPath\": {\"path\": \"${ENCRYPTION_PROVIDER_CONFIG_PATH}\", \"type\": \"File\"}}"
 
     # TODO these are used in other places, convert to global.
     local gce_conf_path="/etc/gce.conf"
@@ -1797,7 +1797,7 @@ EOM
     } " "${src_file}"
   fi
 
-  cp "${src_file}" /etc/kubernetes/manifests
+  cp "${src_file}" "${ETC_MANIFESTS:-/etc/kubernetes/manifests}"
 }
 
 # Starts kubernetes controller manager.
@@ -2488,47 +2488,48 @@ EOF
 }
 
 ########### Main Function ###########
-echo "Start to configure instance for kubernetes"
+function main() {
+  echo "Start to configure instance for kubernetes"
 
-KUBE_HOME="/home/kubernetes"
+  KUBE_HOME="/home/kubernetes"
-CONTAINERIZED_MOUNTER_HOME="${KUBE_HOME}/containerized_mounter"
+  CONTAINERIZED_MOUNTER_HOME="${KUBE_HOME}/containerized_mounter"
-PV_RECYCLER_OVERRIDE_TEMPLATE="${KUBE_HOME}/kube-manifests/kubernetes/pv-recycler-template.yaml"
+  PV_RECYCLER_OVERRIDE_TEMPLATE="${KUBE_HOME}/kube-manifests/kubernetes/pv-recycler-template.yaml"
 
-if [[ ! -e "${KUBE_HOME}/kube-env" ]]; then
+  if [[ ! -e "${KUBE_HOME}/kube-env" ]]; then
     echo "The ${KUBE_HOME}/kube-env file does not exist!! Terminate cluster initialization."
     exit 1
-fi
+  fi
 
-source "${KUBE_HOME}/kube-env"
+  source "${KUBE_HOME}/kube-env"
 
 
-if [[ -f "${KUBE_HOME}/kubelet-config.yaml" ]]; then
+  if [[ -f "${KUBE_HOME}/kubelet-config.yaml" ]]; then
     echo "Found Kubelet config file at ${KUBE_HOME}/kubelet-config.yaml"
     KUBELET_CONFIG_FILE_ARG="--config ${KUBE_HOME}/kubelet-config.yaml"
-fi
+  fi
 
-if [[ -e "${KUBE_HOME}/kube-master-certs" ]]; then
+  if [[ -e "${KUBE_HOME}/kube-master-certs" ]]; then
     source "${KUBE_HOME}/kube-master-certs"
-fi
+  fi
 
-if [[ -n "${KUBE_USER:-}" ]]; then
+  if [[ -n "${KUBE_USER:-}" ]]; then
     if ! [[ "${KUBE_USER}" =~ ^[-._@a-zA-Z0-9]+$ ]]; then
       echo "Bad KUBE_USER format."
       exit 1
     fi
-fi
+  fi
 
-# generate the controller manager and scheduler tokens here since they are only used on the master.
+  # generate the controller manager and scheduler tokens here since they are only used on the master.
-KUBE_CONTROLLER_MANAGER_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null)
+  KUBE_CONTROLLER_MANAGER_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null)
-KUBE_SCHEDULER_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null)
+  KUBE_SCHEDULER_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null)
 
-setup-os-params
+  setup-os-params
-config-ip-firewall
+  config-ip-firewall
-create-dirs
+  create-dirs
-setup-kubelet-dir
+  setup-kubelet-dir
-ensure-local-ssds
+  ensure-local-ssds
-setup-logrotate
+  setup-logrotate
-if [[ "${KUBERNETES_MASTER:-}" == "true" ]]; then
+  if [[ "${KUBERNETES_MASTER:-}" == "true" ]]; then
     mount-master-pd
     create-node-pki
     create-master-pki
@@ -2536,7 +2537,7 @@ if [[ "${KUBERNETES_MASTER:-}" == "true" ]]; then
     create-master-kubelet-auth
     create-master-etcd-auth
     override-pv-recycler
-else
+  else
     create-node-pki
     create-kubelet-kubeconfig ${KUBERNETES_MASTER_NAME}
     if [[ "${KUBE_PROXY_DAEMONSET:-}" != "true" ]]; then
@@ -2545,16 +2546,16 @@ else
     if [[ "${ENABLE_NODE_PROBLEM_DETECTOR:-}" == "standalone" ]]; then
       create-node-problem-detector-kubeconfig
     fi
-fi
+  fi
 
-override-kubectl
+  override-kubectl
-# Run the containerized mounter once to pre-cache the container image.
+  # Run the containerized mounter once to pre-cache the container image.
-if [[ "${CONTAINER_RUNTIME:-docker}" == "docker" ]]; then
+  if [[ "${CONTAINER_RUNTIME:-docker}" == "docker" ]]; then
     assemble-docker-flags
-fi
+  fi
-start-kubelet
+  start-kubelet
 
-if [[ "${KUBERNETES_MASTER:-}" == "true" ]]; then
+  if [[ "${KUBERNETES_MASTER:-}" == "true" ]]; then
     compute-master-manifest-variables
     start-etcd-servers
     start-etcd-empty-dir-cleanup-pod
@@ -2565,7 +2566,7 @@ if [[ "${KUBERNETES_MASTER:-}" == "true" ]]; then
     start-cluster-autoscaler
     start-lb-controller
     start-rescheduler
-else
+  else
     if [[ "${KUBE_PROXY_DAEMONSET:-}" != "true" ]]; then
       start-kube-proxy
     fi
@@ -2575,8 +2576,16 @@ else
     if [[ "${ENABLE_NODE_PROBLEM_DETECTOR:-}" == "standalone" ]]; then
       start-node-problem-detector
     fi
+  fi
+  reset-motd
+  prepare-mounter-rootfs
+  modprobe configs
+  echo "Done for the configuration for kubernetes"
+}
+
+# use --source-only to test functions defined in this script.
+if [[ "$#" -eq 1 && "${1}" == "--source-only" ]]; then
+   :
+else
+   main "${@}"
 fi
-reset-motd
-prepare-mounter-rootfs
-modprobe configs
-echo "Done for the configuration for kubernetes"

cluster/gce/gci/configure_helper_test.go

@@ -0,0 +1,172 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package gci
+
+import (
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"testing"
+	"text/template"
+
+	"k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/kubernetes/pkg/api/legacyscheme"
+)
+
+const (
+	envScriptFileName         = "kube-env"
+	configureHelperScriptName = "configure-helper.sh"
+)
+
+type ManifestTestCase struct {
+	pod                 v1.Pod
+	envScriptPath       string
+	manifest            string
+	auxManifests        []string
+	kubeHome            string
+	manifestSources     string
+	manifestDestination string
+	manifestTemplateDir string
+	manifestTemplate    string
+	manifestFuncName    string
+	t                   *testing.T
+}
+
+func newManifestTestCase(t *testing.T, manifest, funcName string, auxManifests []string) *ManifestTestCase {
+	c := &ManifestTestCase{
+		t:                t,
+		manifest:         manifest,
+		auxManifests:     auxManifests,
+		manifestFuncName: funcName,
+	}
+
+	d, err := ioutil.TempDir("", "configure-helper-test")
+	if err != nil {
+		c.t.Fatalf("Failed to create temp directory: %v", err)
+	}
+
+	c.kubeHome = d
+	c.envScriptPath = filepath.Join(c.kubeHome, envScriptFileName)
+	c.manifestSources = filepath.Join(c.kubeHome, "kube-manifests", "kubernetes", "gci-trusty")
+
+	currentPath, err := os.Getwd()
+	if err != nil {
+		c.t.Fatalf("Failed to get current directory: %v", err)
+	}
+	gceDir := filepath.Dir(currentPath)
+	c.manifestTemplateDir = filepath.Join(gceDir, "manifests")
+	c.manifestTemplate = filepath.Join(c.manifestTemplateDir, c.manifest)
+	c.manifestDestination = filepath.Join(c.kubeHome, "etc", "kubernetes", "manifests", c.manifest)
+
+	c.mustCopyFromTemplate()
+	c.mustCopyAuxFromTemplate()
+	c.mustCreateManifestDstDir()
+
+	return c
+}
+
+func (c *ManifestTestCase) mustCopyFromTemplate() {
+	if err := os.MkdirAll(c.manifestSources, os.ModePerm); err != nil {
+		c.t.Fatalf("Failed to create source directory: %v", err)
+	}
+
+	if err := copyFile(c.manifestTemplate, filepath.Join(c.manifestSources, c.manifest)); err != nil {
+		c.t.Fatalf("Failed to copy source manifest to KUBE_HOME: %v", err)
+	}
+}
+
+func (c *ManifestTestCase) mustCopyAuxFromTemplate() {
+	for _, m := range c.auxManifests {
+		err := copyFile(filepath.Join(c.manifestTemplateDir, m), filepath.Join(c.manifestSources, m))
+		if err != nil {
+			c.t.Fatalf("Failed to copy source manifest %s to KUBE_HOME: %v", m, err)
+		}
+	}
+}
+
+func (c *ManifestTestCase) mustCreateManifestDstDir() {
+	p := filepath.Join(filepath.Join(c.kubeHome, "etc", "kubernetes", "manifests"))
+	if err := os.MkdirAll(p, os.ModePerm); err != nil {
+		c.t.Fatalf("Failed to create designation folder for kube-apiserver.manifest: %v", err)
+	}
+}
+
+func (c *ManifestTestCase) mustCreateEnv(envTemplate string, env interface{}) {
+	f, err := os.Create(filepath.Join(c.kubeHome, envScriptFileName))
+	if err != nil {
+		c.t.Fatalf("Failed to create envScript: %v", err)
+	}
+	defer f.Close()
+
+	t := template.Must(template.New("env").Parse(envTemplate))
+
+	if err = t.Execute(f, env); err != nil {
+		c.t.Fatalf("Failed to execute template: %v", err)
+	}
+}
+
+func (c *ManifestTestCase) mustInvokeFunc(envTemplate string, env interface{}) {
+	c.mustCreateEnv(envTemplate, env)
+	args := fmt.Sprintf("source %s ; source %s --source-only ; %s", c.envScriptPath, configureHelperScriptName, c.manifestFuncName)
+	cmd := exec.Command("bash", "-c", args)
+
+	bs, err := cmd.CombinedOutput()
+	if err != nil {
+		c.t.Logf("%s", bs)
+		c.t.Fatalf("Failed to run configure-helper.sh: %v", err)
+	}
+	c.t.Logf("%s", string(bs))
+}
+
+func (c *ManifestTestCase) mustLoadPodFromManifest() {
+	json, err := ioutil.ReadFile(c.manifestDestination)
+	if err != nil {
+		c.t.Fatalf("Failed to read manifest: %s, %v", c.manifestDestination, err)
+	}
+
+	if err := runtime.DecodeInto(legacyscheme.Codecs.UniversalDecoder(), json, &c.pod); err != nil {
+		c.t.Fatalf("Failed to decode manifest: %v", err)
+	}
+}
+
+func (c *ManifestTestCase) tearDown() {
+	os.RemoveAll(c.kubeHome)
+}
+
+func copyFile(src, dst string) (err error) {
+	in, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+	defer in.Close()
+	out, err := os.Create(dst)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		cerr := out.Close()
+		if cerr == nil {
+			err = cerr
+		}
+	}()
+	_, err = io.Copy(out, in)
+	return err
+}

cluster/gce/manifests/BUILD

@@ -0,0 +1,55 @@
+package(default_visibility = ["//visibility:public"])
+
+load("@io_kubernetes_build//defs:build.bzl", "release_filegroup")
+load("@io_kubernetes_build//defs:pkg.bzl", "pkg_tar")
+
+pkg_tar(
+    name = "gce-master-manifests",
+    srcs = [
+        "abac-authz-policy.jsonl",
+        "cluster-autoscaler.manifest",
+        "e2e-image-puller.manifest",
+        "etcd.manifest",
+        "glbc.manifest",
+        "kms-plugin-container.manifest",
+        "kube-addon-manager.yaml",
+        "kube-apiserver.manifest",
+        "kube-controller-manager.manifest",
+        "kube-proxy.manifest",
+        "kube-scheduler.manifest",
+        "rescheduler.manifest",
+    ],
+    mode = "0644",
+)
+
+filegroup(
+    name = "manifests-test-data",
+    srcs = [
+        "abac-authz-policy.jsonl",
+        "cluster-autoscaler.manifest",
+        "e2e-image-puller.manifest",
+        "etcd.manifest",
+        "glbc.manifest",
+        "kms-plugin-container.manifest",
+        "kube-addon-manager.yaml",
+        "kube-apiserver.manifest",
+        "kube-controller-manager.manifest",
+        "kube-proxy.manifest",
+        "kube-scheduler.manifest",
+        "rescheduler.manifest",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)