Add unit test for configure-helper.

build/root/BUILD.root

@@ -28,13 +28,13 @@ gcs_upload(
     data = [
         ":_binary-artifacts-and-hashes",
         "//build/release-tars:release-tars-and-hashes",
-        "//cluster/gce:gcs-release-artifacts-and-hashes",
+        "//cluster/gce/gci:gcs-release-artifacts-and-hashes",
     ],
     tags = ["manual"],
     upload_paths = {
         "//:_binary-artifacts-and-hashes": "bin/linux/amd64",
         "//build/release-tars:release-tars-and-hashes": "",
-        "//cluster/gce:gcs-release-artifacts-and-hashes": "extra/gce",
+        "//cluster/gce/gci:gcs-release-artifacts-and-hashes": "extra/gce",
     },
 )
 

cluster/BUILD

@@ -29,9 +29,9 @@ pkg_tar(
     package_dir = "kubernetes/gci-trusty",
     deps = [
         "//cluster/addons",
-        "//cluster/gce:gce-master-manifests",
-        "//cluster/gce:gci-trusty-manifests",
         "//cluster/gce/addons",
+        "//cluster/gce/gci:gci-trusty-manifests",
+        "//cluster/gce/manifests:gce-master-manifests",
     ],
 )
 

cluster/gce/BUILD

@@ -3,17 +3,6 @@ package(default_visibility = ["//visibility:public"])
 load("@io_kubernetes_build//defs:build.bzl", "release_filegroup")
 load("@io_kubernetes_build//defs:pkg.bzl", "pkg_tar")
 
-pkg_tar(
-    name = "gci-trusty-manifests",
-    files = {
-        "//cluster/gce/gci/mounter": "gci-mounter",
-        "gci/configure-helper.sh": "gci-configure-helper.sh",
-        "gci/health-monitor.sh": "health-monitor.sh",
-    },
-    mode = "0755",
-    strip_prefix = ".",
-)
-
 filegroup(
     name = "package-srcs",
     srcs = glob(["**"]),
@@ -26,38 +15,8 @@ filegroup(
     srcs = [
         ":package-srcs",
         "//cluster/gce/addons:all-srcs",
-        "//cluster/gce/gci/mounter:all-srcs",
+        "//cluster/gce/gci:all-srcs",
+        "//cluster/gce/manifests:all-srcs",
     ],
     tags = ["automanaged"],
 )
-
-# Having the COS code from the GCE cluster deploy hosted with the release is
-# useful for GKE.  This list should match the list in
-# kubernetes/release/lib/releaselib.sh.
-release_filegroup(
-    name = "gcs-release-artifacts",
-    srcs = [
-        "gci/configure.sh",
-        "gci/master.yaml",
-        "gci/node.yaml",
-    ],
-)
-
-pkg_tar(
-    name = "gce-master-manifests",
-    srcs = [
-        "manifests/abac-authz-policy.jsonl",
-        "manifests/cluster-autoscaler.manifest",
-        "manifests/e2e-image-puller.manifest",
-        "manifests/etcd.manifest",
-        "manifests/glbc.manifest",
-        "manifests/kms-plugin-container.manifest",
-        "manifests/kube-addon-manager.yaml",
-        "manifests/kube-apiserver.manifest",
-        "manifests/kube-controller-manager.manifest",
-        "manifests/kube-proxy.manifest",
-        "manifests/kube-scheduler.manifest",
-        "manifests/rescheduler.manifest",
-    ],
-    mode = "0644",
-)

cluster/gce/gci/BUILD

@@ -0,0 +1,69 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_test")
+load("@io_kubernetes_build//defs:pkg.bzl", "pkg_tar")
+load("@io_kubernetes_build//defs:build.bzl", "release_filegroup")
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "apiserver_manifest_test.go",
+        "configure_helper_test.go",
+    ],
+    data = [
+        ":scripts-test-data",
+        "//cluster/gce/manifests:manifests-test-data",
+    ],
+    deps = [
+        "//pkg/api/legacyscheme:go_default_library",
+        "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
+    ],
+)
+
+# Having the COS code from the GCE cluster deploy hosted with the release is
+# useful for GKE.  This list should match the list in
+# kubernetes/release/lib/releaselib.sh.
+release_filegroup(
+    name = "gcs-release-artifacts",
+    srcs = [
+        "configure.sh",
+        "master.yaml",
+        "node.yaml",
+    ],
+    visibility = ["//visibility:public"],
+)
+
+pkg_tar(
+    name = "gci-trusty-manifests",
+    files = {
+        "//cluster/gce/gci/mounter": "gci-mounter",
+        "configure-helper.sh": "gci-configure-helper.sh",
+        "health-monitor.sh": "health-monitor.sh",
+    },
+    mode = "0755",
+    strip_prefix = ".",
+    visibility = ["//visibility:public"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//cluster/gce/gci/mounter:all-srcs",
+    ],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)
+
+filegroup(
+    name = "scripts-test-data",
+    srcs = [
+        "configure-helper.sh",
+    ],
+)

cluster/gce/gci/apiserver_manifest_test.go

@@ -0,0 +1,212 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package gci
+
+import (
+	"encoding/base64"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"k8s.io/api/core/v1"
+)
+
+const (
+	/*
+	   Template for defining the environment state of configure-helper.sh
+	   The environment of configure-helper.sh is initially configured via kube-env file. However, as deploy-helper
+	   executes new variables are created. ManifestTestCase does not care where a variable came from. However, future
+	   test scenarios, may require such a distinction.
+	   The list of variables is, by no means, complete - this is what is required to run currently defined tests.
+	*/
+	deployHelperEnv = `
+readonly KUBE_HOME={{.KubeHome}}
+readonly KUBE_API_SERVER_LOG_PATH=${KUBE_HOME}/kube-apiserver.log
+readonly KUBE_API_SERVER_AUDIT_LOG_PATH=${KUBE_HOME}/kube-apiserver-audit.log
+readonly CLOUD_CONFIG_OPT=--cloud-config=/etc/gce.conf
+readonly CA_CERT_BUNDLE_PATH=/foo/bar
+readonly APISERVER_SERVER_CERT_PATH=/foo/bar
+readonly APISERVER_SERVER_KEY_PATH=/foo/bar
+readonly APISERVER_CLIENT_CERT_PATH=/foo/bar
+readonly CLOUD_CONFIG_MOUNT="{\"name\": \"cloudconfigmount\",\"mountPath\": \"/etc/gce.conf\", \"readOnly\": true},"
+readonly CLOUD_CONFIG_VOLUME="{\"name\": \"cloudconfigmount\",\"hostPath\": {\"path\": \"/etc/gce.conf\", \"type\": \"FileOrCreate\"}},"
+readonly DOCKER_REGISTRY="k8s.gcr.io"
+readonly ENABLE_LEGACY_ABAC=false
+readonly ETC_MANIFESTS=${KUBE_HOME}/etc/kubernetes/manifests
+readonly KUBE_API_SERVER_DOCKER_TAG=v1.11.0-alpha.0.1808_3c7452dc11645d-dirty
+readonly LOG_OWNER_USER=$(whoami)
+readonly LOG_OWNER_GROUP=$(id -gn)
+ENCRYPTION_PROVIDER_CONFIG={{.EncryptionProviderConfig}}
+ENCRYPTION_PROVIDER_CONFIG_PATH={{.EncryptionProviderConfigPath}}
+readonly ETCD_KMS_KEY_ID={{.ETCDKMSKeyID}}
+`
+	kubeAPIServerManifestFileName = "kube-apiserver.manifest"
+	kmsPluginManifestFileName     = "kms-plugin-container.manifest"
+	kubeAPIServerStartFuncName    = "start-kube-apiserver"
+
+	// Position of containers within a pod manifest
+	kmsPluginContainerIndex        = 0
+	apiServerContainerIndexNoKMS   = 0
+	apiServerContainerIndexWithKMS = 1
+
+	//	command": [
+	//   "/bin/sh", - Index 0
+	//   "-c",      - Index 1
+	//   "exec /usr/local/bin/kube-apiserver " - Index 2
+	execArgsIndex = 2
+
+	socketVolumeMountIndexKMSPlugin = 1
+	socketVolumeMountIndexAPIServer = 0
+)
+
+type kubeAPIServerEnv struct {
+	KubeHome                     string
+	EncryptionProviderConfig     string
+	EncryptionProviderConfigPath string
+	ETCDKMSKeyID                 string
+}
+
+type kubeAPIServerManifestTestCase struct {
+	*ManifestTestCase
+	apiServerContainer v1.Container
+	kmsPluginContainer v1.Container
+}
+
+func newKubeAPIServerManifestTestCase(t *testing.T) *kubeAPIServerManifestTestCase {
+	return &kubeAPIServerManifestTestCase{
+		ManifestTestCase: newManifestTestCase(t, kubeAPIServerManifestFileName, kubeAPIServerStartFuncName, []string{kmsPluginManifestFileName}),
+	}
+}
+
+func (c *kubeAPIServerManifestTestCase) mustLoadContainers() {
+	c.mustLoadPodFromManifest()
+
+	switch len(c.pod.Spec.Containers) {
+	case 1:
+		c.apiServerContainer = c.pod.Spec.Containers[apiServerContainerIndexNoKMS]
+	case 2:
+		c.apiServerContainer = c.pod.Spec.Containers[apiServerContainerIndexWithKMS]
+		c.kmsPluginContainer = c.pod.Spec.Containers[kmsPluginContainerIndex]
+	default:
+		c.t.Fatalf("got %d containers in apiserver pod, want 1 or 2", len(c.pod.Spec.Containers))
+	}
+}
+
+func (c *kubeAPIServerManifestTestCase) invokeTest(e kubeAPIServerEnv) {
+	c.mustInvokeFunc(deployHelperEnv, e)
+	c.mustLoadContainers()
+}
+
+func getEncryptionProviderConfigFlag(path string) string {
+	return fmt.Sprintf("--experimental-encryption-provider-config=%s", path)
+}
+
+func TestEncryptionProviderFlag(t *testing.T) {
+	c := newKubeAPIServerManifestTestCase(t)
+	defer c.tearDown()
+
+	e := kubeAPIServerEnv{
+		KubeHome:                     c.kubeHome,
+		EncryptionProviderConfig:     base64.StdEncoding.EncodeToString([]byte("FOO")),
+		EncryptionProviderConfigPath: filepath.Join(c.kubeHome, "encryption-provider-config.yaml"),
+	}
+
+	c.invokeTest(e)
+
+	expectedFlag := getEncryptionProviderConfigFlag(e.EncryptionProviderConfigPath)
+	execArgs := c.apiServerContainer.Command[execArgsIndex]
+	if !strings.Contains(execArgs, expectedFlag) {
+		c.t.Fatalf("Got %q, wanted the flag to contain %q", execArgs, expectedFlag)
+	}
+}
+
+func TestEncryptionProviderConfig(t *testing.T) {
+	c := newKubeAPIServerManifestTestCase(t)
+	defer c.tearDown()
+
+	p := filepath.Join(c.kubeHome, "encryption-provider-config.yaml")
+	e := kubeAPIServerEnv{
+		KubeHome:                     c.kubeHome,
+		EncryptionProviderConfig:     base64.StdEncoding.EncodeToString([]byte("FOO")),
+		EncryptionProviderConfigPath: p,
+	}
+
+	c.mustInvokeFunc(deployHelperEnv, e)
+
+	if _, err := os.Stat(p); err != nil {
+		c.t.Fatalf("Expected encryption provider config to be written to %s, but stat failed with error: %v", p, err)
+	}
+}
+
+// TestKMSEncryptionProviderConfig asserts that if ETCD_KMS_KEY_ID is set then start-kube-apiserver will produce
+// EncryptionProviderConfig file of type KMS and inject experimental-encryption-provider-config startup flag.
+func TestKMSEncryptionProviderConfig(t *testing.T) {
+	c := newKubeAPIServerManifestTestCase(t)
+	defer c.tearDown()
+
+	e := kubeAPIServerEnv{
+		KubeHome:                     c.kubeHome,
+		EncryptionProviderConfigPath: filepath.Join(c.kubeHome, "encryption-provider-config.yaml"),
+		ETCDKMSKeyID:                 "FOO",
+	}
+
+	c.invokeTest(e)
+
+	expectedFlag := getEncryptionProviderConfigFlag(e.EncryptionProviderConfigPath)
+	execArgs := c.apiServerContainer.Command[execArgsIndex]
+	if !strings.Contains(execArgs, expectedFlag) {
+		c.t.Fatalf("Got %q, wanted the flag to contain %q", execArgs, expectedFlag)
+	}
+
+	p := filepath.Join(c.kubeHome, "encryption-provider-config.yaml")
+	if _, err := os.Stat(p); err != nil {
+		c.t.Fatalf("Expected encryption provider config to be written to %s, but stat failed with error: %v", p, err)
+	}
+
+	d, err := ioutil.ReadFile(p)
+	if err != nil {
+		c.t.Fatalf("Failed to read encryption provider config %s", p)
+	}
+
+	if !strings.Contains(string(d), "name: grpc-kms-provider") {
+		c.t.Fatalf("Got %s\n, wanted encryption provider config to be of type grpc-kms", string(d))
+	}
+}
+
+func TestKMSPluginAndAPIServerSharedVolume(t *testing.T) {
+	c := newKubeAPIServerManifestTestCase(t)
+	defer c.tearDown()
+
+	var e = kubeAPIServerEnv{
+		KubeHome:                     c.kubeHome,
+		EncryptionProviderConfigPath: filepath.Join(c.kubeHome, "encryption-provider-config.yaml"),
+		ETCDKMSKeyID:                 "FOO",
+	}
+
+	c.invokeTest(e)
+
+	k := c.kmsPluginContainer.VolumeMounts[socketVolumeMountIndexKMSPlugin].MountPath
+	a := c.apiServerContainer.VolumeMounts[socketVolumeMountIndexAPIServer].MountPath
+
+	if k != a {
+		t.Fatalf("Got %s!=%s, wanted KMSPlugin VolumeMount #1:%s to be equal to kube-apiserver VolumeMount #0:%s",
+			k, a, k, a)
+	}
+}

cluster/gce/gci/configure-helper.sh

@@ -1177,7 +1177,7 @@ EOF
 function prepare-log-file {
   touch $1
   chmod 644 $1
-  chown root:root $1
+  chown "${LOG_OWNER_USER:-root}":"${LOG_OWNER_GROUP:-root}" $1
 }
 
 # Prepares parameters for kube-proxy manifest.
@@ -1402,8 +1402,8 @@ function prepare-mounter-rootfs {
 #   DOCKER_REGISTRY
 function start-kube-apiserver {
   echo "Start kubernetes api-server"
-  prepare-log-file /var/log/kube-apiserver.log
+  prepare-log-file "${KUBE_API_SERVER_LOG_PATH:-/var/log/kube-apiserver.log}"
-  prepare-log-file /var/log/kube-apiserver-audit.log
+  prepare-log-file "${KUBE_API_SERVER_AUDIT_LOG_PATH:-/var/log/kube-apiserver-audit.log}"
 
   # Calculate variables and assemble the command line.
   local params="${API_SERVER_TEST_LOG_LEVEL:-"--v=2"} ${APISERVER_TEST_ARGS:-} ${CLOUD_CONFIG_OPT}"
@@ -1700,14 +1700,14 @@ EOM
   fi
 
   if [[ -n "${ENCRYPTION_PROVIDER_CONFIG:-}" ]]; then
-    local encryption_provider_config_path="/etc/srv/kubernetes/encryption-provider-config.yml"
+    ENCRYPTION_PROVIDER_CONFIG_PATH="${ENCRYPTION_PROVIDER_CONFIG_PATH:-/etc/srv/kubernetes/encryption-provider-config.yml}"
-    echo "${ENCRYPTION_PROVIDER_CONFIG}" | base64 --decode > "${encryption_provider_config_path}"
+    echo "${ENCRYPTION_PROVIDER_CONFIG}" | base64 --decode > "${ENCRYPTION_PROVIDER_CONFIG_PATH}"
-    params+=" --experimental-encryption-provider-config=${encryption_provider_config_path}"
+    params+=" --experimental-encryption-provider-config=${ENCRYPTION_PROVIDER_CONFIG_PATH}"
   fi
 
   src_file="${src_dir}/kube-apiserver.manifest"
   # Evaluate variables.
-  local -r kube_apiserver_docker_tag=$(cat /home/kubernetes/kube-docker-files/kube-apiserver.docker_tag)
+  local -r kube_apiserver_docker_tag="${KUBE_API_SERVER_DOCKER_TAG:-$(cat /home/kubernetes/kube-docker-files/kube-apiserver.docker_tag)}"
   sed -i -e "s@{{params}}@${params}@g" "${src_file}"
   sed -i -e "s@{{container_env}}@${container_env}@g" ${src_file}
   sed -i -e "s@{{srv_kube_path}}@/etc/srv/kubernetes@g" "${src_file}"
@@ -1754,8 +1754,8 @@ EOM
       exit 1
     fi
 
-    if [[ ! -f "${encryption_provider_config_path}" ]]; then
+    if [[ ! -f "${ENCRYPTION_PROVIDER_CONFIG_PATH}" ]]; then
-      echo "Error: KMS Integration was requested, but "${encryption_provider_config_path}" is missing."
+      echo "Error: KMS Integration was requested, but "${ENCRYPTION_PROVIDER_CONFIG_PATH}" is missing."
       exit 1
     fi
 
@@ -1769,8 +1769,8 @@ EOM
     local kms_socket_vol="{ \"name\": \"kmssocket\", \"hostPath\": {\"path\": \"${kms_socket_dir}\", \"type\": \"DirectoryOrCreate\"}}"
     local kms_path_to_socket="${kms_socket_dir}/socket.sock"
 
-    local encryption_provider_mnt="{ \"name\": \"encryptionconfig\", \"mountPath\": \"${encryption_provider_config_path}\", \"readOnly\": true}"
+    local encryption_provider_mnt="{ \"name\": \"encryptionconfig\", \"mountPath\": \"${ENCRYPTION_PROVIDER_CONFIG_PATH}\", \"readOnly\": true}"
-    local encryption_provider_vol="{ \"name\": \"encryptionconfig\", \"hostPath\": {\"path\": \"${encryption_provider_config_path}\", \"type\": \"File\"}}"
+    local encryption_provider_vol="{ \"name\": \"encryptionconfig\", \"hostPath\": {\"path\": \"${ENCRYPTION_PROVIDER_CONFIG_PATH}\", \"type\": \"File\"}}"
 
     # TODO these are used in other places, convert to global.
     local gce_conf_path="/etc/gce.conf"
@@ -1795,7 +1795,7 @@ EOM
     } " "${src_file}"
   fi
 
-  cp "${src_file}" /etc/kubernetes/manifests
+  cp "${src_file}" "${ETC_MANIFESTS:-/etc/kubernetes/manifests}"
 }
 
 # Starts kubernetes controller manager.
@@ -2473,95 +2473,104 @@ EOF
 }
 
 ########### Main Function ###########
-echo "Start to configure instance for kubernetes"
+function main() {
+  echo "Start to configure instance for kubernetes"
 
-KUBE_HOME="/home/kubernetes"
+  KUBE_HOME="/home/kubernetes"
-CONTAINERIZED_MOUNTER_HOME="${KUBE_HOME}/containerized_mounter"
+  CONTAINERIZED_MOUNTER_HOME="${KUBE_HOME}/containerized_mounter"
-PV_RECYCLER_OVERRIDE_TEMPLATE="${KUBE_HOME}/kube-manifests/kubernetes/pv-recycler-template.yaml"
+  PV_RECYCLER_OVERRIDE_TEMPLATE="${KUBE_HOME}/kube-manifests/kubernetes/pv-recycler-template.yaml"
 
-if [[ ! -e "${KUBE_HOME}/kube-env" ]]; then
+  if [[ ! -e "${KUBE_HOME}/kube-env" ]]; then
-  echo "The ${KUBE_HOME}/kube-env file does not exist!! Terminate cluster initialization."
+    echo "The ${KUBE_HOME}/kube-env file does not exist!! Terminate cluster initialization."
-  exit 1
-fi
-
-source "${KUBE_HOME}/kube-env"
-
-
-if [[ -f "${KUBE_HOME}/kubelet-config.yaml" ]]; then
-  echo "Found Kubelet config file at ${KUBE_HOME}/kubelet-config.yaml"
-  KUBELET_CONFIG_FILE_ARG="--config ${KUBE_HOME}/kubelet-config.yaml"
-fi
-
-if [[ -e "${KUBE_HOME}/kube-master-certs" ]]; then
-  source "${KUBE_HOME}/kube-master-certs"
-fi
-
-if [[ -n "${KUBE_USER:-}" ]]; then
-  if ! [[ "${KUBE_USER}" =~ ^[-._@a-zA-Z0-9]+$ ]]; then
-    echo "Bad KUBE_USER format."
     exit 1
   fi
-fi
 
-# generate the controller manager and scheduler tokens here since they are only used on the master.
+  source "${KUBE_HOME}/kube-env"
-KUBE_CONTROLLER_MANAGER_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null)
-KUBE_SCHEDULER_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null)
 
-setup-os-params
+
-config-ip-firewall
+  if [[ -f "${KUBE_HOME}/kubelet-config.yaml" ]]; then
-create-dirs
+    echo "Found Kubelet config file at ${KUBE_HOME}/kubelet-config.yaml"
-setup-kubelet-dir
+    KUBELET_CONFIG_FILE_ARG="--config ${KUBE_HOME}/kubelet-config.yaml"
-ensure-local-ssds
+  fi
-setup-logrotate
+
-if [[ "${KUBERNETES_MASTER:-}" == "true" ]]; then
+  if [[ -e "${KUBE_HOME}/kube-master-certs" ]]; then
-  mount-master-pd
+    source "${KUBE_HOME}/kube-master-certs"
-  create-node-pki
+  fi
-  create-master-pki
+
-  create-master-auth
+  if [[ -n "${KUBE_USER:-}" ]]; then
-  create-master-kubelet-auth
+    if ! [[ "${KUBE_USER}" =~ ^[-._@a-zA-Z0-9]+$ ]]; then
-  create-master-etcd-auth
+      echo "Bad KUBE_USER format."
-  override-pv-recycler
+      exit 1
+    fi
+  fi
+
+  # generate the controller manager and scheduler tokens here since they are only used on the master.
+  KUBE_CONTROLLER_MANAGER_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null)
+  KUBE_SCHEDULER_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null)
+
+  setup-os-params
+  config-ip-firewall
+  create-dirs
+  setup-kubelet-dir
+  ensure-local-ssds
+  setup-logrotate
+  if [[ "${KUBERNETES_MASTER:-}" == "true" ]]; then
+    mount-master-pd
+    create-node-pki
+    create-master-pki
+    create-master-auth
+    create-master-kubelet-auth
+    create-master-etcd-auth
+    override-pv-recycler
+  else
+    create-node-pki
+    create-kubelet-kubeconfig ${KUBERNETES_MASTER_NAME}
+    if [[ "${KUBE_PROXY_DAEMONSET:-}" != "true" ]]; then
+      create-kubeproxy-user-kubeconfig
+    fi
+    if [[ "${ENABLE_NODE_PROBLEM_DETECTOR:-}" == "standalone" ]]; then
+      create-node-problem-detector-kubeconfig
+    fi
+  fi
+
+  override-kubectl
+  # Run the containerized mounter once to pre-cache the container image.
+  if [[ "${CONTAINER_RUNTIME:-docker}" == "docker" ]]; then
+    assemble-docker-flags
+  fi
+  start-kubelet
+
+  if [[ "${KUBERNETES_MASTER:-}" == "true" ]]; then
+    compute-master-manifest-variables
+    start-etcd-servers
+    start-etcd-empty-dir-cleanup-pod
+    start-kube-apiserver
+    start-kube-controller-manager
+    start-kube-scheduler
+    start-kube-addons
+    start-cluster-autoscaler
+    start-lb-controller
+    start-rescheduler
+  else
+    if [[ "${KUBE_PROXY_DAEMONSET:-}" != "true" ]]; then
+      start-kube-proxy
+    fi
+    if [[ "${PREPULL_E2E_IMAGES:-}" == "true" ]]; then
+      start-image-puller
+    fi
+    if [[ "${ENABLE_NODE_PROBLEM_DETECTOR:-}" == "standalone" ]]; then
+      start-node-problem-detector
+    fi
+  fi
+  reset-motd
+  prepare-mounter-rootfs
+  modprobe configs
+  echo "Done for the configuration for kubernetes"
+}
+
+# use --source-only to test functions defined in this script.
+if [[ "$#" -eq 1 && "${1}" == "--source-only" ]]; then
+   :
 else
-  create-node-pki
+   main "${@}"
-  create-kubelet-kubeconfig ${KUBERNETES_MASTER_NAME}
+fi
-  if [[ "${KUBE_PROXY_DAEMONSET:-}" != "true" ]]; then
-    create-kubeproxy-user-kubeconfig
-  fi
-  if [[ "${ENABLE_NODE_PROBLEM_DETECTOR:-}" == "standalone" ]]; then
-    create-node-problem-detector-kubeconfig
-  fi
-fi
-
-override-kubectl
-# Run the containerized mounter once to pre-cache the container image.
-if [[ "${CONTAINER_RUNTIME:-docker}" == "docker" ]]; then
-  assemble-docker-flags
-fi
-start-kubelet
-
-if [[ "${KUBERNETES_MASTER:-}" == "true" ]]; then
-  compute-master-manifest-variables
-  start-etcd-servers
-  start-etcd-empty-dir-cleanup-pod
-  start-kube-apiserver
-  start-kube-controller-manager
-  start-kube-scheduler
-  start-kube-addons
-  start-cluster-autoscaler
-  start-lb-controller
-  start-rescheduler
-else
-  if [[ "${KUBE_PROXY_DAEMONSET:-}" != "true" ]]; then
-    start-kube-proxy
-  fi
-  if [[ "${PREPULL_E2E_IMAGES:-}" == "true" ]]; then
-    start-image-puller
-  fi
-  if [[ "${ENABLE_NODE_PROBLEM_DETECTOR:-}" == "standalone" ]]; then
-    start-node-problem-detector
-  fi
-fi
-reset-motd
-prepare-mounter-rootfs
-modprobe configs
-echo "Done for the configuration for kubernetes"

cluster/gce/gci/configure_helper_test.go

@@ -0,0 +1,172 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package gci
+
+import (
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"testing"
+	"text/template"
+
+	"k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/kubernetes/pkg/api/legacyscheme"
+)
+
+const (
+	envScriptFileName         = "kube-env"
+	configureHelperScriptName = "configure-helper.sh"
+)
+
+type ManifestTestCase struct {
+	pod                 v1.Pod
+	envScriptPath       string
+	manifest            string
+	auxManifests        []string
+	kubeHome            string
+	manifestSources     string
+	manifestDestination string
+	manifestTemplateDir string
+	manifestTemplate    string
+	manifestFuncName    string
+	t                   *testing.T
+}
+
+func newManifestTestCase(t *testing.T, manifest, funcName string, auxManifests []string) *ManifestTestCase {
+	c := &ManifestTestCase{
+		t:                t,
+		manifest:         manifest,
+		auxManifests:     auxManifests,
+		manifestFuncName: funcName,
+	}
+
+	d, err := ioutil.TempDir("", "configure-helper-test")
+	if err != nil {
+		c.t.Fatalf("Failed to create temp directory: %v", err)
+	}
+
+	c.kubeHome = d
+	c.envScriptPath = filepath.Join(c.kubeHome, envScriptFileName)
+	c.manifestSources = filepath.Join(c.kubeHome, "kube-manifests", "kubernetes", "gci-trusty")
+
+	currentPath, err := os.Getwd()
+	if err != nil {
+		c.t.Fatalf("Failed to get current directory: %v", err)
+	}
+	gceDir := filepath.Dir(currentPath)
+	c.manifestTemplateDir = filepath.Join(gceDir, "manifests")
+	c.manifestTemplate = filepath.Join(c.manifestTemplateDir, c.manifest)
+	c.manifestDestination = filepath.Join(c.kubeHome, "etc", "kubernetes", "manifests", c.manifest)
+
+	c.mustCopyFromTemplate()
+	c.mustCopyAuxFromTemplate()
+	c.mustCreateManifestDstDir()
+
+	return c
+}
+
+func (c *ManifestTestCase) mustCopyFromTemplate() {
+	if err := os.MkdirAll(c.manifestSources, os.ModePerm); err != nil {
+		c.t.Fatalf("Failed to create source directory: %v", err)
+	}
+
+	if err := copyFile(c.manifestTemplate, filepath.Join(c.manifestSources, c.manifest)); err != nil {
+		c.t.Fatalf("Failed to copy source manifest to KUBE_HOME: %v", err)
+	}
+}
+
+func (c *ManifestTestCase) mustCopyAuxFromTemplate() {
+	for _, m := range c.auxManifests {
+		err := copyFile(filepath.Join(c.manifestTemplateDir, m), filepath.Join(c.manifestSources, m))
+		if err != nil {
+			c.t.Fatalf("Failed to copy source manifest %s to KUBE_HOME: %v", m, err)
+		}
+	}
+}
+
+func (c *ManifestTestCase) mustCreateManifestDstDir() {
+	p := filepath.Join(filepath.Join(c.kubeHome, "etc", "kubernetes", "manifests"))
+	if err := os.MkdirAll(p, os.ModePerm); err != nil {
+		c.t.Fatalf("Failed to create designation folder for kube-apiserver.manifest: %v", err)
+	}
+}
+
+func (c *ManifestTestCase) mustCreateEnv(envTemplate string, env interface{}) {
+	f, err := os.Create(filepath.Join(c.kubeHome, envScriptFileName))
+	if err != nil {
+		c.t.Fatalf("Failed to create envScript: %v", err)
+	}
+	defer f.Close()
+
+	t := template.Must(template.New("env").Parse(envTemplate))
+
+	if err = t.Execute(f, env); err != nil {
+		c.t.Fatalf("Failed to execute template: %v", err)
+	}
+}
+
+func (c *ManifestTestCase) mustInvokeFunc(envTemplate string, env interface{}) {
+	c.mustCreateEnv(envTemplate, env)
+	args := fmt.Sprintf("source %s ; source %s --source-only ; %s", c.envScriptPath, configureHelperScriptName, c.manifestFuncName)
+	cmd := exec.Command("bash", "-c", args)
+
+	bs, err := cmd.CombinedOutput()
+	if err != nil {
+		c.t.Logf("%s", bs)
+		c.t.Fatalf("Failed to run configure-helper.sh: %v", err)
+	}
+	c.t.Logf("%s", string(bs))
+}
+
+func (c *ManifestTestCase) mustLoadPodFromManifest() {
+	json, err := ioutil.ReadFile(c.manifestDestination)
+	if err != nil {
+		c.t.Fatalf("Failed to read manifest: %s, %v", c.manifestDestination, err)
+	}
+
+	if err := runtime.DecodeInto(legacyscheme.Codecs.UniversalDecoder(), json, &c.pod); err != nil {
+		c.t.Fatalf("Failed to decode manifest: %v", err)
+	}
+}
+
+func (c *ManifestTestCase) tearDown() {
+	os.RemoveAll(c.kubeHome)
+}
+
+func copyFile(src, dst string) (err error) {
+	in, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+	defer in.Close()
+	out, err := os.Create(dst)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		cerr := out.Close()
+		if cerr == nil {
+			err = cerr
+		}
+	}()
+	_, err = io.Copy(out, in)
+	return err
+}

cluster/gce/manifests/BUILD

@@ -0,0 +1,55 @@
+package(default_visibility = ["//visibility:public"])
+
+load("@io_kubernetes_build//defs:build.bzl", "release_filegroup")
+load("@io_kubernetes_build//defs:pkg.bzl", "pkg_tar")
+
+pkg_tar(
+    name = "gce-master-manifests",
+    srcs = [
+        "abac-authz-policy.jsonl",
+        "cluster-autoscaler.manifest",
+        "e2e-image-puller.manifest",
+        "etcd.manifest",
+        "glbc.manifest",
+        "kms-plugin-container.manifest",
+        "kube-addon-manager.yaml",
+        "kube-apiserver.manifest",
+        "kube-controller-manager.manifest",
+        "kube-proxy.manifest",
+        "kube-scheduler.manifest",
+        "rescheduler.manifest",
+    ],
+    mode = "0644",
+)
+
+filegroup(
+    name = "manifests-test-data",
+    srcs = [
+        "abac-authz-policy.jsonl",
+        "cluster-autoscaler.manifest",
+        "e2e-image-puller.manifest",
+        "etcd.manifest",
+        "glbc.manifest",
+        "kms-plugin-container.manifest",
+        "kube-addon-manager.yaml",
+        "kube-apiserver.manifest",
+        "kube-controller-manager.manifest",
+        "kube-proxy.manifest",
+        "kube-scheduler.manifest",
+        "rescheduler.manifest",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)