Initial Kubemark commit

cluster/gce/config-default.sh

@@ -48,8 +48,8 @@ RUNTIME_CONFIG="${KUBE_RUNTIME_CONFIG:-}"
 ENABLE_EXPERIMENTAL_API="${KUBE_ENABLE_EXPERIMENTAL_API:-false}"
 
 # Increase the sleep interval value if concerned about API rate limits. 3, in seconds, is the default.
-POLL_SLEEP_INTERVAL=3
+POLL_SLEEP_INTERVAL="${POLL_SLEEP_INTERVAL:-3}"
-SERVICE_CLUSTER_IP_RANGE="10.0.0.0/16"  # formerly PORTAL_NET
+SERVICE_CLUSTER_IP_RANGE="${SERVICE_CLUSTER_IP_RANGE:-10.0.0.0/16}"  # formerly PORTAL_NET
 ALLOCATE_NODE_CIDRS=true
 
 # Optional: Cluster monitoring to setup as part of the cluster bring up:

cluster/images/kubemark/Dockerfile

@@ -0,0 +1,7 @@
+FROM debian:jessie
+
+COPY kubemark.sh /kubemark.sh
+COPY kubernetes-server-linux-amd64.tar.gz /tmp/kubemark.tar.gz
+COPY build-kubemark.sh /build-kubemark.sh
+
+RUN /build-kubemark.sh

cluster/images/kubemark/Makefile

@@ -0,0 +1,9 @@
+# build Kubemark image from currently built binaries containing both 'real' master and Hollow Node
+
+all:
+	cp ../../../_output/release-tars/kubernetes-server-linux-amd64.tar.gz .
+	docker build -t gcr.io/$(PROJECT)/kubemark .
+	gcloud docker push gcr.io/$(PROJECT)/kubemark
+	rm kubernetes-server-linux-amd64.tar.gz
+
+.PHONY: all

cluster/images/kubemark/build-kubemark.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+
+# Copyright 2015 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# We don't need all intermediate steps in the image, especially because
+# we clean after ourselves. Thus instead of doing all of this in the Dockerfile
+# we use this script.
+apt-get update
+apt-get install -y wget vim rsync ca-certificates
+update-ca-certificates
+
+chmod a+x /kubemark.sh
+
+tar xzf /tmp/kubemark.tar.gz
+cp kubernetes/server/bin/hyperkube /
+cp kubernetes/server/bin/kubemark /
+cp kubernetes/server/bin/kubectl /
+
+rm -rf /tmp/*
+apt-get remove -y build-essential 
+apt-get clean -y
+apt-get autoremove -y

cluster/images/kubemark/kubemark.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+# Copyright 2015 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Currently populating the Pod IP thought the downward API is broken (#13690)
+# this is a temporary workaround.
+# TODO: get rid of this script when downward API is fixed.
+./kubemark --name=`hostname -i` $@

cluster/kubemark/config-default.sh

@@ -0,0 +1,66 @@
+#!/bin/bash
+
+# Copyright 2015 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# A configuration for Kubemark cluster. It doesn't need to be kept in
+# sync with gce/config-default.sh (except the filename, because I'm reusing 
+# gce/util.sh script which assumes config filename), but if some things that
+# are enabled by default should not run in hollow clusters, they should be disabled here.
+
+GCLOUD=gcloud
+ZONE=${KUBE_GCE_ZONE:-us-central1-b}
+MASTER_SIZE=${MASTER_SIZE:-n1-standard-4}
+NUM_MINIONS=${NUM_MINIONS:-100}
+MASTER_DISK_TYPE=pd-ssd
+MASTER_DISK_SIZE=${MASTER_DISK_SIZE:-20GB}
+REGISTER_MASTER_KUBELET=${REGISTER_MASTER:-false}
+PREEMPTIBLE_MINION=${PREEMPTIBLE_MINION:-false}
+
+OS_DISTRIBUTION=${KUBE_OS_DISTRIBUTION:-debian}
+MASTER_IMAGE=${KUBE_GCE_MASTER_IMAGE:-container-vm-v20150806}
+MASTER_IMAGE_PROJECT=${KUBE_GCE_MASTER_PROJECT:-google-containers}
+
+NETWORK=${KUBE_GCE_NETWORK:-default}
+INSTANCE_PREFIX="hollow-kubernetes"
+MASTER_NAME="hollow-cluster-master"
+MASTER_TAG="hollow-cluster-master"
+MASTER_IP_RANGE="${MASTER_IP_RANGE:-10.246.0.0/24}"
+CLUSTER_IP_RANGE="${CLUSTER_IP_RANGE:-10.244.0.0/16}"
+RUNTIME_CONFIG="${KUBE_RUNTIME_CONFIG:-}"
+ENABLE_EXPERIMENTAL_API="${KUBE_ENABLE_EXPERIMENTAL_API:-false}"
+
+# Increase the sleep interval value if concerned about API rate limits. 3, in seconds, is the default.
+POLL_SLEEP_INTERVAL=3
+SERVICE_CLUSTER_IP_RANGE="10.0.0.0/16"  # formerly PORTAL_NET
+ALLOCATE_NODE_CIDRS=true
+
+ENABLE_CLUSTER_MONITORING="${KUBE_ENABLE_CLUSTER_MONITORING:-none}"
+ENABLE_NODE_LOGGING="${KUBE_ENABLE_NODE_LOGGING:-false}"
+ENABLE_CLUSTER_LOGGING="${KUBE_ENABLE_CLUSTER_LOGGING:-false}"
+# Optional: Don't require https for registries in our local RFC1918 network
+if [[ ${KUBE_ENABLE_INSECURE_REGISTRY:-false} == "true" ]]; then
+  EXTRA_DOCKER_OPTS="--insecure-registry 10.0.0.0/8"
+fi
+
+ENABLE_CLUSTER_DNS="${KUBE_ENABLE_CLUSTER_DNS:-false}"
+ENABLE_CLUSTER_REGISTRY="${KUBE_ENABLE_CLUSTER_REGISTRY:-false}"
+ENABLE_CLUSTER_UI="${KUBE_ENABLE_CLUSTER_UI:-false}"
+ENABLE_NODE_AUTOSCALER="${KUBE_ENABLE_NODE_AUTOSCALER:-false}"
+
+# Admission Controllers to invoke prior to persisting objects in cluster
+ADMISSION_CONTROL=NamespaceLifecycle,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota
+
+# Optional: if set to true kube-up will automatically check for existing resources and clean them up.
+KUBE_UP_AUTOMATIC_CLEANUP=${KUBE_UP_AUTOMATIC_CLEANUP:-false}

cluster/kubemark/util.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+# Copyright 2014 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../..
+source ${KUBE_ROOT}/cluster/kubemark/config-default.sh
+source ${KUBE_ROOT}/cluster/gce/util.sh

cmd/integration/integration.go

@@ -231,7 +231,8 @@ func startComponents(firstManifestURL, secondManifestURL string) (string, string
 		1*time.Second,  /* HTTPCheckFrequency */
 		10*time.Second, /* MinimumGCAge */
 		3*time.Second,  /* NodeStatusUpdateFrequency */
-		10*time.Second /* SyncFrequency */)
+		10*time.Second, /* SyncFrequency */
+		40 /* MaxPods */)
 
 	kubeletapp.RunKubelet(kcfg, nil)
 	// Kubelet (machine)
@@ -261,7 +262,8 @@ func startComponents(firstManifestURL, secondManifestURL string) (string, string
 		1*time.Second,  /* HTTPCheckFrequency */
 		10*time.Second, /* MinimumGCAge */
 		3*time.Second,  /* NodeStatusUpdateFrequency */
-		10*time.Second /* SyncFrequency */)
+		10*time.Second, /* SyncFrequency */
+		40 /* MaxPods */)
 
 	kubeletapp.RunKubelet(kcfg, nil)
 	return apiServer.URL, configFilePath

cmd/kubelet/app/server.go

@@ -599,7 +599,9 @@ func SimpleKubelet(client *client.Client,
 	configFilePath string,
 	cloud cloudprovider.Interface,
 	osInterface kubecontainer.OSInterface,
-	fileCheckFrequency, httpCheckFrequency, minimumGCAge, nodeStatusUpdateFrequency, syncFrequency time.Duration) *KubeletConfig {
+	fileCheckFrequency, httpCheckFrequency, minimumGCAge, nodeStatusUpdateFrequency, syncFrequency time.Duration,
+	maxPods int,
+) *KubeletConfig {
 
 	imageGCPolicy := kubelet.ImageGCPolicy{
 		HighThresholdPercent: 90,
@@ -634,7 +636,7 @@ func SimpleKubelet(client *client.Client,
 		MaxContainerCount:         100,
 		MaxOpenFiles:              1024,
 		MaxPerPodContainerCount:   2,
-		MaxPods:                   32,
+		MaxPods:                   maxPods,
 		MinimumGCAge:              minimumGCAge,
 		Mounter:                   mount.New(),
 		NodeStatusUpdateFrequency: nodeStatusUpdateFrequency,

cmd/kubemark/hollow-node.go

@@ -0,0 +1,112 @@
+/*
+Copyright 2015 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"runtime"
+	"time"
+
+	docker "github.com/fsouza/go-dockerclient"
+	kubeletapp "k8s.io/kubernetes/cmd/kubelet/app"
+	"k8s.io/kubernetes/pkg/api"
+	"k8s.io/kubernetes/pkg/api/latest"
+	client "k8s.io/kubernetes/pkg/client/unversioned"
+	"k8s.io/kubernetes/pkg/kubelet/cadvisor"
+	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
+	"k8s.io/kubernetes/pkg/kubelet/dockertools"
+	"k8s.io/kubernetes/pkg/util"
+	"k8s.io/kubernetes/pkg/volume/empty_dir"
+
+	"github.com/golang/glog"
+	"github.com/spf13/pflag"
+)
+
+var (
+	fakeDockerClient dockertools.FakeDockerClient
+
+	apiServer           string
+	kubeletPort         int
+	kubeletReadOnlyPort int
+	nodeName            string
+	serverPort          int
+)
+
+func addFlags(fs *pflag.FlagSet) {
+	fs.StringVar(&apiServer, "server", "", "API server IP.")
+	fs.IntVar(&kubeletPort, "kubelet-port", 10250, "Port on which HollowKubelet should be listening.")
+	fs.IntVar(&kubeletReadOnlyPort, "kubelet-read-only-port", 10255, "Read-only port on which Kubelet is listening.")
+	fs.StringVar(&nodeName, "name", "fake-node", "Name of this Hollow Node.")
+	fs.IntVar(&serverPort, "api-server-port", 443, "Port on which API server is listening.")
+}
+
+func makeTempDirOrDie(prefix string, baseDir string) string {
+	if baseDir == "" {
+		baseDir = "/tmp"
+	}
+	tempDir, err := ioutil.TempDir(baseDir, prefix)
+	if err != nil {
+		glog.Fatalf("Can't make a temp rootdir: %v", err)
+	}
+	if err = os.MkdirAll(tempDir, 0750); err != nil {
+		glog.Fatalf("Can't mkdir(%q): %v", tempDir, err)
+	}
+	return tempDir
+}
+
+func main() {
+	runtime.GOMAXPROCS(runtime.NumCPU())
+	addFlags(pflag.CommandLine)
+	util.InitFlags()
+
+	// create a client for Kubelet to communicate with API server.
+	cl := client.NewOrDie(&client.Config{Host: fmt.Sprintf("http://%v:%v", apiServer, serverPort), Version: latest.GroupOrDie("").Version})
+	cadvisorInterface := new(cadvisor.Fake)
+
+	testRootDir := makeTempDirOrDie("hollow-kubelet.", "")
+	configFilePath := makeTempDirOrDie("config", testRootDir)
+	glog.Infof("Using %s as root dir for hollow-kubelet", testRootDir)
+	fakeDockerClient.VersionInfo = docker.Env{"ApiVersion=1.18"}
+	kcfg := kubeletapp.SimpleKubelet(
+		cl,
+		&fakeDockerClient,
+		nodeName,
+		testRootDir,
+		"",        /* manifest-url */
+		"0.0.0.0", /* bind address */
+		uint(kubeletPort),
+		uint(kubeletReadOnlyPort),
+		api.NamespaceDefault,
+		empty_dir.ProbeVolumePlugins(),
+		nil, /* tls-options */
+		cadvisorInterface,
+		configFilePath,
+		nil, /* cloud-provider */
+		kubecontainer.FakeOS{}, /* os-interface */
+		20*time.Second,         /* FileCheckFrequency */
+		20*time.Second,         /* HTTPCheckFrequency */
+		1*time.Minute,          /* MinimumGCAge */
+		10*time.Second,         /* NodeStatusUpdateFrequency */
+		10*time.Second,         /* SyncFrequency */
+		40,                     /* MaxPods */
+	)
+	kubeletapp.RunKubelet(kcfg, nil)
+
+	select {}
+}

hack/lib/golang.sh

@@ -32,6 +32,7 @@ kube::golang::server_targets() {
     cmd/kube-apiserver
     cmd/kube-controller-manager
     cmd/kubelet
+    cmd/kubemark
     cmd/hyperkube
     cmd/linkcheck
     plugin/cmd/kube-scheduler

hack/verify-flags/known-flags.txt

@@ -14,6 +14,7 @@ api-burst
 api-prefix
 api-rate
 api-servers
+api-server-port
 api-token
 api-version
 authorization-mode
@@ -137,6 +138,7 @@ kubelet-https
 kubelet-network-plugin
 kubelet-pod-infra-container-image
 kubelet-port
+kubelet-read-only-port
 kubelet-root-dir
 kubelet-sync-frequency
 kubelet-timeout

test/kubemark/configure-kubectl.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+# Copyright 2015 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+curl https://sdk.cloud.google.com | bash
+sudo gcloud components update kubectl -q
+sudo ln -s /usr/local/share/google/google-cloud-sdk/bin/kubectl /bin/
+kubectl config set-cluster hollow-cluster --server=http://localhost:8080 --insecure-skip-tls-verify=true
+kubectl config set-credentials `whoami`
+kubectl config set-context hollow-context --cluster=hollow-cluster --user=`whoami`

test/kubemark/get-real-pod-for-hollow-node.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+# Copyright 2015 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+kubectl get pod --namespace=kubemark -o json | grep -C100 $1 | grep hollow | head -n1 | cut -d: -f2 | cut -d, -f1 | tr -d \" | tr -d '[[:space:]]'
+echo ""

test/kubemark/hollow-kubelet_template.json

@@ -0,0 +1,54 @@
+{
+	"kind": "ReplicationController",
+	"apiVersion": "v1",
+	"metadata": {
+		"name": "hollow-kubelet",
+		"labels": {
+			"name": "hollow-kubelet"
+		}
+	},
+	"spec": {
+		"replicas": ##numreplicas##,
+		"selector": {
+			"name": "hollow-kubelet"
+		},
+		"template": {
+		  "metadata": {
+				"labels": {
+					"name": "hollow-kubelet"
+				}
+			},
+			"spec": {
+				"containers": [{
+					"name": "hollow-kubelet",
+					"image": "gcr.io/gmarek-kubernetes/kubemark:latest",
+					"ports": [
+						{"containerPort": 4194},
+						{"containerPort": 10250},
+						{"containerPort": 10255}
+					],
+					"env": [
+						{
+							"name": "MASTER_IP",
+							"value": ##masterip##
+						}
+					],
+					"command": [
+						"./kubemark.sh"
+					],
+					"args": [
+						"--server=$(MASTER_IP)",
+						"--api-server-port=8080",
+						"--v=3"
+					],
+					"resources": {
+						"requests": {
+							"cpu": "50m",
+							"memory": "100M"
+						}
+					}
+				}]
+			}
+		}
+	}
+}

test/kubemark/kubemark-ns.json

@@ -0,0 +1,7 @@
+{
+	"kind": "Namespace",
+	"apiVersion": "v1",
+	"metadata": {
+		"name": "kubemark"
+	}
+}

test/kubemark/run-scalability-test.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+# Copyright 2015 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../..
+
+export KUBERNETES_PROVIDER="kubemark"
+export KUBE_CONFIG_FILE="config-default.sh"
+
+source ${KUBE_ROOT}/cluster/kubemark/util.sh
+source ${KUBE_ROOT}/cluster/kubemark/config-default.sh
+
+echo ${KUBERNETES_PROVIDER}
+echo ${MASTER_NAME}
+
+detect-master
+
+export KUBE_MASTER_URL="http://${KUBE_MASTER_IP:-}:8080"
+
+${KUBE_ROOT}/hack/ginkgo-e2e.sh --e2e-verify-service-account=false --ginkgo.focus="should\sallow\sstarting\s30\spods\sper\snode"

test/kubemark/start-kubemark-master.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+# Copyright 2015 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# TODO: figure out how to get etcd tag from some real configuration and put it here.
+sudo docker run --net=host -d gcr.io/google_containers/etcd:2.0.12 /usr/local/bin/etcd --addr=127.0.0.1:4001 --bind-addr=0.0.0.0:4001 --data-dir=/var/etcd/data
+
+# Increase the allowed number of open file descriptors
++ulimit -n 65536
+
+tar xzf kubernetes-server-linux-amd64.tar.gz
+
+kubernetes/server/bin/kube-controller-manager --master=127.0.0.1:8080 --v=2 &> /tmp/kube-controller-manager.log &
+
+kubernetes/server/bin/kube-scheduler --master=127.0.0.1:8080 --v=2 &> /tmp/kube-scheduler.log &
+
+kubernetes/server/bin/kube-apiserver --portal-net=10.0.0.1/24 --address=0.0.0.0 --etcd-servers=http://127.0.0.1:4001 --cluster-name=hollow-kubernetes --v=2 &> /tmp/kube-apiserver.log &
+
+rm -rf kubernetes

test/kubemark/start-kubemark.sh

@@ -0,0 +1,67 @@
+#!/bin/bash
+
+# Copyright 2015 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Script that creates a Kubemark cluster with Master running on GCE.
+KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../..
+
+source "${KUBE_ROOT}/cluster/kubemark/config-default.sh"
+source "${KUBE_ROOT}/cluster/kubemark/util.sh"
+
+detect-project &> /dev/null
+export PROJECT
+
+CURR_DIR=`pwd`
+cd ${KUBE_ROOT}/cluster/images/kubemark
+make
+cd $CURR_DIR
+
+MASTER_NAME="hollow-cluster-master"
+
+gcloud compute disks create "${MASTER_NAME}-pd" \
+    --project "${PROJECT}" \
+    --zone "${ZONE}" \
+    --type "${MASTER_DISK_TYPE}" \
+    --size "${MASTER_DISK_SIZE}"
+
+gcloud compute instances create ${MASTER_NAME} \
+    --project "${PROJECT}" \
+    --zone "${ZONE}" \
+    --machine-type "${MASTER_SIZE}" \
+    --image-project="${MASTER_IMAGE_PROJECT}" \
+    --image "${MASTER_IMAGE}" \
+    --tags "${MASTER_TAG}" \
+    --network "${NETWORK}" \
+    --scopes "storage-ro,compute-rw,logging-write" \
+    --disk "name=${MASTER_NAME}-pd,device-name=master-pd,mode=rw,boot=no,auto-delete=no"
+
+MASTER_IP=`gcloud compute instances describe hollow-cluster-master | grep networkIP | cut -f2 -d":" | sed "s/ //g"`
+
+until gcloud compute ssh hollow-cluster-master --command="ls" &> /dev/null; do
+  sleep 1
+done
+
+gcloud compute copy-files \
+  ${KUBE_ROOT}/_output/release-tars/kubernetes-server-linux-amd64.tar.gz \
+  ${KUBE_ROOT}/test/kubemark/start-kubemark-master.sh \
+  ${KUBE_ROOT}/test/kubemark/configure-kubectl.sh \
+  hollow-cluster-master:~
+
+gcloud compute ssh hollow-cluster-master --command="chmod a+x configure-kubectl.sh && chmod a+x start-kubemark-master.sh && ./start-kubemark-master.sh"
+
+sed "s/##masterip##/\"${MASTER_IP}\"/g" ${KUBE_ROOT}/test/kubemark/hollow-kubelet_template.json > ${KUBE_ROOT}/test/kubemark/hollow-kubelet.json
+sed -i'' -e "s/##numreplicas##/${NUM_MINIONS:-10}/g" ${KUBE_ROOT}/test/kubemark/hollow-kubelet.json
+kubectl create -f ${KUBE_ROOT}/test/kubemark/kubemark-ns.json
+kubectl create -f ${KUBE_ROOT}/test/kubemark/hollow-kubelet.json --namespace="kubemark"

test/kubemark/stop-kubemark.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+# Copyright 2015 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Script that destroys Kubemark clusters and deletes all GCE resources created for Master
+KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../..
+
+source "${KUBE_ROOT}/cluster/kubemark/config-default.sh"
+source "${KUBE_ROOT}/cluster/kubemark/util.sh"
+
+detect-project &> /dev/null
+
+MASTER_NAME="hollow-cluster-master"
+
+kubectl delete -f ${KUBE_ROOT}/test/kubemark/hollow-kubelet.json &> /dev/null || true
+kubectl delete -f ${KUBE_ROOT}/test/kubemark/kubemark-ns.json &> /dev/null || true
+
+gcloud compute instances delete ${MASTER_NAME} \
+    --project "${PROJECT}" \
+    --quiet \
+    --zone "${ZONE}" || true
+
+gcloud compute disks delete \
+      --project "${PROJECT}" \
+      --quiet \
+      --zone "${ZONE}" \
+      "${MASTER_NAME}"-pd