Bring up a cluster using coreos image for worker nodes.

cluster/gce/configure-vm.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-set -x
+
 # Copyright 2015 Google Inc. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

cluster/gce/coreos/helper.sh

@@ -0,0 +1,127 @@
+#!/bin/bash
+
+# Copyright 2015 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# A library of helper functions and constant for coreos os distro
+
+# $1: if 'true', we're building a master yaml, else a node
+function build-kube-env {
+  local master=$1
+  local file=$2
+
+  rm -f ${file}
+  # TODO(dawnchen): master node is still running with debian image
+  if [[ "${master}" == "true" ]]; then
+  cat >$file <<EOF
+ENV_TIMESTAMP: $(yaml-quote $(date -u +%Y-%m-%dT%T%z))
+INSTANCE_PREFIX: $(yaml-quote ${INSTANCE_PREFIX})
+NODE_INSTANCE_PREFIX: $(yaml-quote ${NODE_INSTANCE_PREFIX})
+SERVER_BINARY_TAR_URL: $(yaml-quote ${SERVER_BINARY_TAR_URL})
+SALT_TAR_URL: $(yaml-quote ${SALT_TAR_URL})
+PORTAL_NET: $(yaml-quote ${PORTAL_NET})
+ENABLE_CLUSTER_MONITORING: $(yaml-quote ${ENABLE_CLUSTER_MONITORING:-false})
+ENABLE_NODE_MONITORING: $(yaml-quote ${ENABLE_NODE_MONITORING:-false})
+ENABLE_CLUSTER_LOGGING: $(yaml-quote ${ENABLE_CLUSTER_LOGGING:-false})
+ENABLE_NODE_LOGGING: $(yaml-quote ${ENABLE_NODE_LOGGING:-false})
+LOGGING_DESTINATION: $(yaml-quote ${LOGGING_DESTINATION:-})
+ELASTICSEARCH_LOGGING_REPLICAS: $(yaml-quote ${ELASTICSEARCH_LOGGING_REPLICAS:-})
+ENABLE_CLUSTER_DNS: $(yaml-quote ${ENABLE_CLUSTER_DNS:-false})
+DNS_REPLICAS: $(yaml-quote ${DNS_REPLICAS:-})
+DNS_SERVER_IP: $(yaml-quote ${DNS_SERVER_IP:-})
+DNS_DOMAIN: $(yaml-quote ${DNS_DOMAIN:-})
+KUBE_USER: $(yaml-quote ${KUBE_USER})
+KUBE_PASSWORD: $(yaml-quote ${KUBE_PASSWORD})
+KUBE_BEARER_TOKEN: $(yaml-quote ${KUBE_BEARER_TOKEN})
+KUBELET_TOKEN: $(yaml-quote ${KUBELET_TOKEN:-})
+KUBE_PROXY_TOKEN: $(yaml-quote ${KUBE_PROXY_TOKEN:-})
+ADMISSION_CONTROL: $(yaml-quote ${ADMISSION_CONTROL:-})
+MASTER_IP_RANGE: $(yaml-quote ${MASTER_IP_RANGE})
+EOF
+  else
+    cat >>$file <<EOF
+ENV_TIMESTAMP=$(yaml-quote $(date -u +%Y-%m-%dT%T%z))
+INSTANCE_PREFIX=$(yaml-quote ${INSTANCE_PREFIX})
+NODE_INSTANCE_PREFIX=$(yaml-quote ${NODE_INSTANCE_PREFIX})
+SERVER_BINARY_TAR_URL=$(yaml-quote ${SERVER_BINARY_TAR_URL})
+PORTAL_NET=$(yaml-quote ${PORTAL_NET})
+ENABLE_CLUSTER_MONITORING=$(yaml-quote ${ENABLE_CLUSTER_MONITORING:-false})
+ENABLE_NODE_MONITORING=$(yaml-quote ${ENABLE_NODE_MONITORING:-false})
+ENABLE_CLUSTER_LOGGING=$(yaml-quote ${ENABLE_CLUSTER_LOGGING:-false})
+ENABLE_NODE_LOGGING=$(yaml-quote ${ENABLE_NODE_LOGGING:-false})
+LOGGING_DESTINATION=$(yaml-quote ${LOGGING_DESTINATION:-})
+ELASTICSEARCH_LOGGING_REPLICAS=$(yaml-quote ${ELASTICSEARCH_LOGGING_REPLICAS:-})
+ENABLE_CLUSTER_DNS=$(yaml-quote ${ENABLE_CLUSTER_DNS:-false})
+DNS_REPLICAS=$(yaml-quote ${DNS_REPLICAS:-})
+DNS_SERVER_IP=$(yaml-quote ${DNS_SERVER_IP:-})
+DNS_DOMAIN=$(yaml-quote ${DNS_DOMAIN:-})
+KUBE_USER=$(yaml-quote ${KUBE_USER})
+KUBE_PASSWORD=$(yaml-quote ${KUBE_PASSWORD})
+KUBE_BEARER_TOKEN=$(yaml-quote ${KUBE_BEARER_TOKEN})
+KUBELET_TOKEN=$(yaml-quote ${KUBELET_TOKEN:-})
+KUBE_PROXY_TOKEN=$(yaml-quote ${KUBE_PROXY_TOKEN:-})
+ADMISSION_CONTROL=$(yaml-quote ${ADMISSION_CONTROL:-})
+MASTER_IP_RANGE=$(yaml-quote ${MASTER_IP_RANGE})
+KUBERNETES_MASTER_NAME=$(yaml-quote ${MASTER_NAME})
+ZONE=$(yaml-quote ${ZONE})
+EXTRA_DOCKER_OPTS=$(yaml-quote ${EXTRA_DOCKER_OPTS})
+ENABLE_DOCKER_REGISTRY_CACHE=$(yaml-quote ${ENABLE_DOCKER_REGISTRY_CACHE:-false})
+PROJECT_ID=$(yaml-quote ${PROJECT})
+EOF
+  fi
+}
+
+# create-master-instance creates the master instance. If called with
+# an argument, the argument is used as the name to a reserved IP
+# address for the master. (In the case of upgrade/repair, we re-use
+# the same IP.)
+#
+# It requires a whole slew of assumed variables, partially due to to
+# the call to write-master-env. Listing them would be rather
+# futile. Instead, we list the required calls to ensure any additional
+# variables are set:
+#   ensure-temp-dir
+#   detect-project
+#   get-bearer-token
+#
+# TODO(dawnchen): Convert master node to use coreos image too
+function create-master-instance {
+  local address_opt=""
+  [[ -n ${1:-} ]] && address_opt="--address ${1}"
+
+  write-master-env
+  gcloud compute instances create "${MASTER_NAME}" \
+    ${address_opt} \
+    --project "${PROJECT}" \
+    --zone "${ZONE}" \
+    --machine-type "${MASTER_SIZE}" \
+    --image-project="${MASTER_IMAGE_PROJECT}" \
+    --image "${MASTER_IMAGE}" \
+    --tags "${MASTER_TAG}" \
+    --network "${NETWORK}" \
+    --scopes "storage-ro" "compute-rw" \
+    --can-ip-forward \
+    --metadata-from-file \
+      "startup-script=${KUBE_ROOT}/cluster/gce/configure-vm.sh" \
+      "kube-env=${KUBE_TEMP}/master-kube-env.yaml" \
+    --disk name="${MASTER_NAME}-pd" device-name=master-pd mode=rw boot=no auto-delete=no
+}
+
+# TODO(dawnchen): Check $CONTAINER_RUNTIME to decide which
+# cloud_config yaml file should be passed
+function create-node-instance-template {
+   create-node-template "${NODE_INSTANCE_PREFIX}-template" "${scope_flags[*]}" \
+    "kube-env=${KUBE_TEMP}/node-kube-env.yaml" \
+    "user-data=${KUBE_ROOT}/cluster/gce/coreos/node.yaml"
+}

cluster/gce/coreos/node.yaml

@@ -0,0 +1,156 @@
+#cloud-config
+
+write_files:
+  - path: /run/configure-hostname.sh
+    permissions: "0755"
+    content: |
+      #!/bin/bash -e
+      set -x
+      source /etc/kube-env
+
+      hostnamectl set-hostname $(hostname | cut -f1 -d.)
+  - path: /run/setup-auth.sh
+    permissions: "0755"
+    content: |
+      #!/bin/bash -e
+      set -x
+      source /etc/kube-env
+
+      /usr/bin/mkdir -p /var/lib/kubelet
+      /bin/echo  {\"BearerToken\": \"${KUBE_BEARER_TOKEN}\", \"Insecure\": true } > /var/lib/kubelet/kubernetes_auth
+  - path: /run/config-kube-proxy.sh
+    permissions: "0755"
+    content: |
+      #!/bin/bash -e
+      set -x
+      source /etc/kube-env
+
+      /usr/bin/mkdir -p /var/lib/kube-proxy
+      cat > /var/lib/kube-proxy/kubeconfig << EOF
+      apiVersion: v1
+      kind: Config
+      users:
+      - name: kube-proxy
+        user:
+          token: ${KUBE_PROXY_TOKEN}
+      clusters:
+      - name: local
+        cluster:
+          insecure-skip-tls-verify: true
+      contexts:
+      - context:
+          cluster: local
+          user: kube-proxy
+        name: service-account-context
+      current-context: service-account-context
+      EOF
+
+coreos:
+  units:
+    - name: kube-env.service
+      command: start
+      content: |
+        [Unit]
+        Description=Fetch kubernetes-node-environment
+        Requires=network-online.target
+        After=network-online.target
+        [Service]
+        Type=oneshot
+        RemainAfterExit=yes
+        ExecStart=/usr/bin/curl --fail --silent --show-error \
+        -H "X-Google-Metadata-Request: True" \
+        -o /etc/kube-env \
+        http://metadata.google.internal/computeMetadata/v1/instance/attributes/kube-env
+
+    - name: kubernetes-install-rkt.service
+      command: start
+      content: |
+        [Unit]
+        Description=Fetch Rocket
+        Documentation=http://github.com/coreos/rkt
+        Requires=network-online.target
+        After=network-online.target
+        [Service]
+        EnvironmentFile=/etc/kube-env
+        ExecStartPre=/usr/bin/mkdir -p /opt/rkt
+        ExecStartPre=/usr/bin/wget \
+        -O /opt/rkt/rkt-v0.5.4.tar.gz \
+        https://github.com/coreos/rkt/releases/download/v0.5.4/rkt-v0.5.4.tar.gz
+        ExecStartPre=/usr/bin/tar xzvf /opt/rkt/rkt-v0.5.4.tar.gz -C /opt --overwrite
+        ExecStart=/bin/systemd-run rkt metadata-service
+
+    - name: kubernetes-install-minion.service
+      command: start
+      content: |
+        [Unit]
+        Description=Install Kubernetes Server
+        Requires=network-online.target
+        After=network-online.target
+        Requires=kube-env.service
+        After=kube-env.service
+        [Service]
+        Type=oneshot
+        RemainAfterExit=yes
+        EnvironmentFile=/etc/kube-env
+        ExecStartPre=/usr/bin/mkdir -p /opt/kubernetes/pkg
+        ExecStartPre=/usr/bin/curl --location --create-dirs --output /opt/kubernetes/pkg/kubernetes-server-linux-amd64.tar.gz ${SERVER_BINARY_TAR_URL}
+        ExecStart=/usr/bin/tar xf /opt/kubernetes/pkg/kubernetes-server-linux-amd64.tar.gz -C /opt --overwrite
+
+    - name: kubernetes-preparation.service
+      command: start
+      content: |
+        [Unit]
+        Description=Configure Node For Kubernetes service
+        Requires=kubernetes-install-minion.service
+        After=kubernetes-install-minion.service
+        Requires=kubernetes-install-rkt.service
+        After=kubernetes-install-rkt.service
+        [Service]
+        Type=oneshot
+        RemainAfterExit=yes
+        EnvironmentFile=/etc/kube-env
+        # TODO(dawnchen): Push this to separate write-files
+        ExecStart=/run/configure-hostname.sh
+
+    - name: kubelet.service
+      command: start
+      content: |
+        [Unit]
+        Description=Run Kubelet service
+        Requires=kubernetes-preparation.service
+        After=kubernetes-preparation.service
+        [Service]
+        EnvironmentFile=/etc/kube-env
+        ExecStartPre=/usr/bin/mkdir -p /etc/kubernetes/manifests
+        ExecStartPre=/run/setup-auth.sh
+        ExecStart=/opt/kubernetes/server/bin/kubelet \
+        --api_servers=https://kubernetes-master.c.${PROJECT_ID}.internal \
+        --config=/etc/kubernetes/manifests \
+        --allow_privileged=False \
+        --v=2 \
+        --cluster_dns=10.0.0.10 \
+        --cluster_domain=kubernetes.local \
+        --logtostderr=true
+        Restart=always
+        RestartSec=10
+
+    - name: kube-proxy.service
+      command: start
+      content: |
+        [Unit]
+        Description=Start Kube-proxy service as Daemon
+        Requires=kubernetes-install-minion.service
+        After=kubernetes-install-minion.service
+        Requires=kubernetes-install-rkt.service
+        After=kubernetes-install-rkt.service
+        [Service]
+        EnvironmentFile=/etc/kube-env
+        ExecStartPre=/run/config-kube-proxy.sh
+        ExecStart=/opt/kubernetes/server/bin/kube-proxy \
+        --master=https://kubernetes-master.c.${PROJECT_ID}.internal \
+        --kubeconfig=/var/lib/kube-proxy/kubeconfig \
+        --v=2 \
+        --logtostderr=true
+        Restart=always
+        RestartSec=10
+

cluster/gce/debian/helper.sh

@@ -88,7 +88,7 @@ function create-master-instance {
     --scopes "storage-ro" "compute-rw" \
     --can-ip-forward \
     --metadata-from-file \
-      "startup-script=${KUBE_ROOT}/cluster/gce/debian/configure-vm.sh" \
+      "startup-script=${KUBE_ROOT}/cluster/gce/configure-vm.sh" \
       "kube-env=${KUBE_TEMP}/master-kube-env.yaml" \
     --disk name="${MASTER_NAME}-pd" device-name=master-pd mode=rw boot=no auto-delete=no
 }

cluster/gce/util.sh

@@ -22,9 +22,12 @@ KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../..
 source "${KUBE_ROOT}/cluster/gce/${KUBE_CONFIG_FILE-"config-default.sh"}"
 source "${KUBE_ROOT}/cluster/common.sh"
 
-if [[ "${OS_DISTRIBUTION}" =~ ^"debian" ]]; then
+if [[ "${OS_DISTRIBUTION}" == "debian" || "${OS_DISTRIBUTION}" == "coreos" ]]; then
-  echo "Starting cluster using os distro : ${OS_DISTRIBUTION}" >&2
+  echo "Starting cluster using os distro: ${OS_DISTRIBUTION}" >&2
   source "${KUBE_ROOT}/cluster/gce/${OS_DISTRIBUTION}/helper.sh"
+else
+  echo "Cannot start cluster using os distro: ${OS_DISTRIBUTION}" >&2
+  return
 fi
 
 NODE_INSTANCE_PREFIX="${INSTANCE_PREFIX}-minion"
@@ -571,6 +574,7 @@ function kube-up {
   for (( i=0; i<${#MINION_NAMES[@]}; i++)); do
     create-route "${MINION_NAMES[$i]}" "${MINION_IP_RANGES[$i]}" &
     add-instance-metadata "${MINION_NAMES[$i]}" "node-ip-range=${MINION_IP_RANGES[$i]}" &
+    add-instance-metadata "${MINION_NAMES[$i]}" "node-name=${MINION_NAMES[$i]}" &
 
     if [ $i -ne 0 ] && [ $((i%5)) -eq 0 ]; then
       echo Waiting for a batch of routes at $i...
@@ -725,6 +729,12 @@ function kube-down {
 
 # Update a kubernetes cluster with latest source
 function kube-push {
+  #TODO(dawnchen): figure out how to upgrade coreos node
+  if [[ "${OS_DISTRIBUTION}" != "debian" ]]; then
+    echo "Updating a kubernetes cluster with ${OS_DISTRIBUTION} is not supported yet." >&2
+    return
+  fi
+
   OUTPUT=${KUBE_ROOT}/_output/logs
   mkdir -p ${OUTPUT}