From 8963347b9e27dc20d3e1649ff1f24b5c137bd583 Mon Sep 17 00:00:00 2001 From: Dawn Chen Date: Tue, 28 Apr 2015 00:25:43 -0700 Subject: [PATCH 1/4] Introduce MASTER_IMAGE, MINION_IMAGE and OS_DISTRIBUTION to config-default for enable coreos and rocket support --- cluster/gce/config-default.sh | 13 ++++++++----- cluster/gce/config-test.sh | 13 ++++++++----- cluster/gce/util.sh | 8 ++++---- 3 files changed, 20 insertions(+), 14 deletions(-) diff --git a/cluster/gce/config-default.sh b/cluster/gce/config-default.sh index 2f723af69d5..9564b9032e3 100755 --- a/cluster/gce/config-default.sh +++ b/cluster/gce/config-default.sh @@ -23,11 +23,14 @@ MINION_SIZE=${MINION_SIZE:-n1-standard-1} NUM_MINIONS=${NUM_MINIONS:-4} MINION_DISK_TYPE=pd-standard MINION_DISK_SIZE=${MINION_DISK_SIZE:-100GB} -# TODO(dchen1107): Filed an internal issue to create an alias -# for containervm image, so that gcloud will expand this -# to the latest supported image. -IMAGE=container-vm-v20150317 -IMAGE_PROJECT=google-containers + +OS_DISTRIBUTION=${KUBE_OS_DISTRIBUTION:-debian} +MASTER_IMAGE=${KUBE_GCE_MASTER_IMAGE:-container-vm-v20150317} +MASTER_IMAGE_PROJECT=${KUBE_GCE_MASTER_PROJECT:-google-containers} +MINION_IMAGE=${KUBE_GCE_MINION_IMAGE:-container-vm-v20150317} +MINION_IMAGE_PROJECT=${KUBE_GCE_MINION_PROJECT:-google-containers} +CONTAINER_RUNTIME=${KUBE_CONTAINER_RUNTIME:-docker} + NETWORK=${KUBE_GCE_NETWORK:-default} INSTANCE_PREFIX="${KUBE_GCE_INSTANCE_PREFIX:-kubernetes}" MASTER_NAME="${INSTANCE_PREFIX}-master" diff --git a/cluster/gce/config-test.sh b/cluster/gce/config-test.sh index 41b391f99db..69ea1c01682 100755 --- a/cluster/gce/config-test.sh +++ b/cluster/gce/config-test.sh @@ -23,11 +23,14 @@ MINION_SIZE=${MINION_SIZE:-g1-small} NUM_MINIONS=${NUM_MINIONS:-2} MINION_DISK_TYPE=pd-standard MINION_DISK_SIZE=${MINION_DISK_SIZE:-100GB} -# TODO(dchen1107): Filed an internal issue to create an alias -# for containervm image, so that gcloud will expand this -# to the latest supported image. -IMAGE=container-vm-v20150317 -IMAGE_PROJECT=google-containers + +OS_DISTRIBUTION=${KUBE_OS_DISTRIBUTION:-debian} +MASTER_IMAGE=${KUBE_GCE_MASTER_IMAGE:-container-vm-v20150317} +MASTER_IMAGE_PROJECT=${KUBE_GCE_MASTER_PROJECT:-google-containers} +MINION_IMAGE=${KUBE_GCE_MINION_IMAGE:-container-vm-v20150317} +MINION_IMAGE_PROJECT=${KUBE_GCE_MINION_PROJECT:-google-containers} +CONTAINER_RUNTIME=${KUBE_CONTAINER_RUNTIME:-docker} + NETWORK=${KUBE_GCE_NETWORK:-e2e} INSTANCE_PREFIX="${KUBE_GCE_INSTANCE_PREFIX:-e2e-test-${USER}}" MASTER_NAME="${INSTANCE_PREFIX}-master" diff --git a/cluster/gce/util.sh b/cluster/gce/util.sh index 7b46b2a1344..90bf24259a1 100755 --- a/cluster/gce/util.sh +++ b/cluster/gce/util.sh @@ -367,8 +367,8 @@ function create-node-template { --machine-type "${MINION_SIZE}" \ --boot-disk-type "${MINION_DISK_TYPE}" \ --boot-disk-size "${MINION_DISK_SIZE}" \ - --image-project="${IMAGE_PROJECT}" \ - --image "${IMAGE}" \ + --image-project="${MINION_IMAGE_PROJECT}" \ + --image "${MINION_IMAGE}" \ --tags "${MINION_TAG}" \ --network "${NETWORK}" \ $2 \ @@ -522,8 +522,8 @@ function create-master-instance { --project "${PROJECT}" \ --zone "${ZONE}" \ --machine-type "${MASTER_SIZE}" \ - --image-project="${IMAGE_PROJECT}" \ - --image "${IMAGE}" \ + --image-project="${MASTER_IMAGE_PROJECT}" \ + --image "${MASTER_IMAGE}" \ --tags "${MASTER_TAG}" \ --network "${NETWORK}" \ --scopes "storage-ro" "compute-rw" \ From 5fa11322f86ccbdd30f76d6ab637a03082820ec0 Mon Sep 17 00:00:00 2001 From: Dawn Chen Date: Tue, 28 Apr 2015 01:22:25 -0700 Subject: [PATCH 2/4] Factory out debian e.g. ContainerVM image specific support to its own helper utility library. --- cluster/gce/configure-vm.sh | 2 +- cluster/gce/debian/helper.sh | 100 +++++++++++++++++++++++++++++++++++ cluster/gce/util.sh | 87 +++--------------------------- 3 files changed, 107 insertions(+), 82 deletions(-) create mode 100644 cluster/gce/debian/helper.sh diff --git a/cluster/gce/configure-vm.sh b/cluster/gce/configure-vm.sh index 753cb3de90c..7f274d4bccb 100644 --- a/cluster/gce/configure-vm.sh +++ b/cluster/gce/configure-vm.sh @@ -1,5 +1,5 @@ #!/bin/bash - +set -x # Copyright 2015 Google Inc. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/cluster/gce/debian/helper.sh b/cluster/gce/debian/helper.sh new file mode 100644 index 00000000000..9ee97c81a8f --- /dev/null +++ b/cluster/gce/debian/helper.sh @@ -0,0 +1,100 @@ +#!/bin/bash + +# Copyright 2015 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# A library of helper functions and constant for debian os distro + +# $1: if 'true', we're building a master yaml, else a node +function build-kube-env { + local master=$1 + local file=$2 + + rm -f ${file} + cat >$file <>$file <&2 + source "${KUBE_ROOT}/cluster/gce/${OS_DISTRIBUTION}/helper.sh" +fi + NODE_INSTANCE_PREFIX="${INSTANCE_PREFIX}-minion" KUBE_PROMPT_FOR_UPDATE=y @@ -448,48 +453,6 @@ function yaml-quote { echo "'$(echo "${@}" | sed -e "s/'/''/g")'" } -# $1: if 'true', we're building a master yaml, else a node -function build-kube-env { - local master=$1 - local file=$2 - - rm -f ${file} - cat >$file <>$file < Date: Tue, 28 Apr 2015 08:50:43 -0700 Subject: [PATCH 3/4] Bring up a cluster using coreos image for worker nodes. --- cluster/gce/configure-vm.sh | 2 +- cluster/gce/coreos/helper.sh | 127 ++++++++++++++++++++++++++++ cluster/gce/coreos/node.yaml | 156 +++++++++++++++++++++++++++++++++++ cluster/gce/debian/helper.sh | 2 +- cluster/gce/util.sh | 14 +++- 5 files changed, 297 insertions(+), 4 deletions(-) create mode 100644 cluster/gce/coreos/helper.sh create mode 100644 cluster/gce/coreos/node.yaml diff --git a/cluster/gce/configure-vm.sh b/cluster/gce/configure-vm.sh index 7f274d4bccb..753cb3de90c 100644 --- a/cluster/gce/configure-vm.sh +++ b/cluster/gce/configure-vm.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -x + # Copyright 2015 Google Inc. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/cluster/gce/coreos/helper.sh b/cluster/gce/coreos/helper.sh new file mode 100644 index 00000000000..ffb1b988e01 --- /dev/null +++ b/cluster/gce/coreos/helper.sh @@ -0,0 +1,127 @@ +#!/bin/bash + +# Copyright 2015 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# A library of helper functions and constant for coreos os distro + +# $1: if 'true', we're building a master yaml, else a node +function build-kube-env { + local master=$1 + local file=$2 + + rm -f ${file} + # TODO(dawnchen): master node is still running with debian image + if [[ "${master}" == "true" ]]; then + cat >$file <>$file < /var/lib/kubelet/kubernetes_auth + - path: /run/config-kube-proxy.sh + permissions: "0755" + content: | + #!/bin/bash -e + set -x + source /etc/kube-env + + /usr/bin/mkdir -p /var/lib/kube-proxy + cat > /var/lib/kube-proxy/kubeconfig << EOF + apiVersion: v1 + kind: Config + users: + - name: kube-proxy + user: + token: ${KUBE_PROXY_TOKEN} + clusters: + - name: local + cluster: + insecure-skip-tls-verify: true + contexts: + - context: + cluster: local + user: kube-proxy + name: service-account-context + current-context: service-account-context + EOF + +coreos: + units: + - name: kube-env.service + command: start + content: | + [Unit] + Description=Fetch kubernetes-node-environment + Requires=network-online.target + After=network-online.target + [Service] + Type=oneshot + RemainAfterExit=yes + ExecStart=/usr/bin/curl --fail --silent --show-error \ + -H "X-Google-Metadata-Request: True" \ + -o /etc/kube-env \ + http://metadata.google.internal/computeMetadata/v1/instance/attributes/kube-env + + - name: kubernetes-install-rkt.service + command: start + content: | + [Unit] + Description=Fetch Rocket + Documentation=http://github.com/coreos/rkt + Requires=network-online.target + After=network-online.target + [Service] + EnvironmentFile=/etc/kube-env + ExecStartPre=/usr/bin/mkdir -p /opt/rkt + ExecStartPre=/usr/bin/wget \ + -O /opt/rkt/rkt-v0.5.4.tar.gz \ + https://github.com/coreos/rkt/releases/download/v0.5.4/rkt-v0.5.4.tar.gz + ExecStartPre=/usr/bin/tar xzvf /opt/rkt/rkt-v0.5.4.tar.gz -C /opt --overwrite + ExecStart=/bin/systemd-run rkt metadata-service + + - name: kubernetes-install-minion.service + command: start + content: | + [Unit] + Description=Install Kubernetes Server + Requires=network-online.target + After=network-online.target + Requires=kube-env.service + After=kube-env.service + [Service] + Type=oneshot + RemainAfterExit=yes + EnvironmentFile=/etc/kube-env + ExecStartPre=/usr/bin/mkdir -p /opt/kubernetes/pkg + ExecStartPre=/usr/bin/curl --location --create-dirs --output /opt/kubernetes/pkg/kubernetes-server-linux-amd64.tar.gz ${SERVER_BINARY_TAR_URL} + ExecStart=/usr/bin/tar xf /opt/kubernetes/pkg/kubernetes-server-linux-amd64.tar.gz -C /opt --overwrite + + - name: kubernetes-preparation.service + command: start + content: | + [Unit] + Description=Configure Node For Kubernetes service + Requires=kubernetes-install-minion.service + After=kubernetes-install-minion.service + Requires=kubernetes-install-rkt.service + After=kubernetes-install-rkt.service + [Service] + Type=oneshot + RemainAfterExit=yes + EnvironmentFile=/etc/kube-env + # TODO(dawnchen): Push this to separate write-files + ExecStart=/run/configure-hostname.sh + + - name: kubelet.service + command: start + content: | + [Unit] + Description=Run Kubelet service + Requires=kubernetes-preparation.service + After=kubernetes-preparation.service + [Service] + EnvironmentFile=/etc/kube-env + ExecStartPre=/usr/bin/mkdir -p /etc/kubernetes/manifests + ExecStartPre=/run/setup-auth.sh + ExecStart=/opt/kubernetes/server/bin/kubelet \ + --api_servers=https://kubernetes-master.c.${PROJECT_ID}.internal \ + --config=/etc/kubernetes/manifests \ + --allow_privileged=False \ + --v=2 \ + --cluster_dns=10.0.0.10 \ + --cluster_domain=kubernetes.local \ + --logtostderr=true + Restart=always + RestartSec=10 + + - name: kube-proxy.service + command: start + content: | + [Unit] + Description=Start Kube-proxy service as Daemon + Requires=kubernetes-install-minion.service + After=kubernetes-install-minion.service + Requires=kubernetes-install-rkt.service + After=kubernetes-install-rkt.service + [Service] + EnvironmentFile=/etc/kube-env + ExecStartPre=/run/config-kube-proxy.sh + ExecStart=/opt/kubernetes/server/bin/kube-proxy \ + --master=https://kubernetes-master.c.${PROJECT_ID}.internal \ + --kubeconfig=/var/lib/kube-proxy/kubeconfig \ + --v=2 \ + --logtostderr=true + Restart=always + RestartSec=10 + diff --git a/cluster/gce/debian/helper.sh b/cluster/gce/debian/helper.sh index 9ee97c81a8f..9aa1ae5a434 100644 --- a/cluster/gce/debian/helper.sh +++ b/cluster/gce/debian/helper.sh @@ -88,7 +88,7 @@ function create-master-instance { --scopes "storage-ro" "compute-rw" \ --can-ip-forward \ --metadata-from-file \ - "startup-script=${KUBE_ROOT}/cluster/gce/debian/configure-vm.sh" \ + "startup-script=${KUBE_ROOT}/cluster/gce/configure-vm.sh" \ "kube-env=${KUBE_TEMP}/master-kube-env.yaml" \ --disk name="${MASTER_NAME}-pd" device-name=master-pd mode=rw boot=no auto-delete=no } diff --git a/cluster/gce/util.sh b/cluster/gce/util.sh index edaa1e53f21..f1340ec26fa 100755 --- a/cluster/gce/util.sh +++ b/cluster/gce/util.sh @@ -22,9 +22,12 @@ KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../.. source "${KUBE_ROOT}/cluster/gce/${KUBE_CONFIG_FILE-"config-default.sh"}" source "${KUBE_ROOT}/cluster/common.sh" -if [[ "${OS_DISTRIBUTION}" =~ ^"debian" ]]; then - echo "Starting cluster using os distro : ${OS_DISTRIBUTION}" >&2 +if [[ "${OS_DISTRIBUTION}" == "debian" || "${OS_DISTRIBUTION}" == "coreos" ]]; then + echo "Starting cluster using os distro: ${OS_DISTRIBUTION}" >&2 source "${KUBE_ROOT}/cluster/gce/${OS_DISTRIBUTION}/helper.sh" +else + echo "Cannot start cluster using os distro: ${OS_DISTRIBUTION}" >&2 + return fi NODE_INSTANCE_PREFIX="${INSTANCE_PREFIX}-minion" @@ -571,6 +574,7 @@ function kube-up { for (( i=0; i<${#MINION_NAMES[@]}; i++)); do create-route "${MINION_NAMES[$i]}" "${MINION_IP_RANGES[$i]}" & add-instance-metadata "${MINION_NAMES[$i]}" "node-ip-range=${MINION_IP_RANGES[$i]}" & + add-instance-metadata "${MINION_NAMES[$i]}" "node-name=${MINION_NAMES[$i]}" & if [ $i -ne 0 ] && [ $((i%5)) -eq 0 ]; then echo Waiting for a batch of routes at $i... @@ -725,6 +729,12 @@ function kube-down { # Update a kubernetes cluster with latest source function kube-push { + #TODO(dawnchen): figure out how to upgrade coreos node + if [[ "${OS_DISTRIBUTION}" != "debian" ]]; then + echo "Updating a kubernetes cluster with ${OS_DISTRIBUTION} is not supported yet." >&2 + return + fi + OUTPUT=${KUBE_ROOT}/_output/logs mkdir -p ${OUTPUT} From 876f8beec90cfd73220e16adf9c96b0852c06c13 Mon Sep 17 00:00:00 2001 From: Dawn Chen Date: Tue, 28 Apr 2015 15:49:39 -0700 Subject: [PATCH 4/4] Remove unused node-name attribute --- cluster/gce/util.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/cluster/gce/util.sh b/cluster/gce/util.sh index f1340ec26fa..e6765594a64 100755 --- a/cluster/gce/util.sh +++ b/cluster/gce/util.sh @@ -574,7 +574,6 @@ function kube-up { for (( i=0; i<${#MINION_NAMES[@]}; i++)); do create-route "${MINION_NAMES[$i]}" "${MINION_IP_RANGES[$i]}" & add-instance-metadata "${MINION_NAMES[$i]}" "node-ip-range=${MINION_IP_RANGES[$i]}" & - add-instance-metadata "${MINION_NAMES[$i]}" "node-name=${MINION_NAMES[$i]}" & if [ $i -ne 0 ] && [ $((i%5)) -eq 0 ]; then echo Waiting for a batch of routes at $i...