From 1705f4919faae99dc4d28a5f52b8502d92f7118a Mon Sep 17 00:00:00 2001 From: Yifan Gu Date: Fri, 13 Nov 2015 13:14:08 -0800 Subject: [PATCH] cluster/gce/coreos: Update master and node cloud config. --- cluster/gce/coreos/master.yaml | 188 +++++++++++++++++++++++++++++++++ cluster/gce/coreos/node.yaml | 174 +++++++++++++----------------- 2 files changed, 260 insertions(+), 102 deletions(-) create mode 100644 cluster/gce/coreos/master.yaml diff --git a/cluster/gce/coreos/master.yaml b/cluster/gce/coreos/master.yaml new file mode 100644 index 00000000000..d1a4bd895de --- /dev/null +++ b/cluster/gce/coreos/master.yaml @@ -0,0 +1,188 @@ +#cloud-config + +coreos: + units: + - name: kube-env.service + command: start + content: | + [Unit] + Description=Fetch kubernetes-node-environment + Requires=network-online.target + After=network-online.target + [Service] + Type=oneshot + RemainAfterExit=yes + ExecStartPre=/usr/bin/curl --fail --silent --show-error \ + -H "X-Google-Metadata-Request: True" \ + -o /etc/kube-env.yaml \ + http://metadata.google.internal/computeMetadata/v1/instance/attributes/kube-env + # Transform the yaml to env file. + ExecStartPre=/usr/bin/mv /etc/kube-env.yaml /etc/kube-env + ExecStart=/usr/bin/sed -i "s/: '/=/;s/'$//" /etc/kube-env + + - name: kubernetes-install-rkt.service + command: start + content: | + [Unit] + Description=Fetch rkt + Documentation=http://github.com/coreos/rkt + Requires=network-online.target + After=network-online.target + [Service] + Type=oneshot + RemainAfterExit=yes + EnvironmentFile=/etc/kube-env + ExecStartPre=/usr/bin/mkdir -p /etc/rkt + ExecStartPre=/usr/bin/mkdir -p /opt/downloads + ExecStartPre=/usr/bin/curl --location --create-dirs --output /opt/downloads/rkt.tar.gz https://github.com/coreos/rkt/releases/download/v${RKT_VERSION}/rkt-v${RKT_VERSION}.tar.gz + ExecStart=/usr/bin/tar xf /opt/downloads/rkt.tar.gz -C /opt --overwrite + + - name: kubernetes-download-salt.service + command: start + content: | + [Unit] + Description=Download salt + Requires=network-online.target + After=network-online.target + Requires=kube-env.service + After=kube-env.service + [Service] + Type=oneshot + RemainAfterExit=yes + EnvironmentFile=/etc/kube-env + ExecStartPre=/usr/bin/mkdir -p /opt/downloads + ExecStartPre=/usr/bin/curl --location --create-dirs --output /opt/downloads/kubernetes-salt.tar.gz ${SALT_TAR_URL} + # TODO(yifan): Check hash. + ExecStart=/usr/bin/tar xf /opt/downloads/kubernetes-salt.tar.gz -C /opt --overwrite + + - name: kubernetes-download-manifests.service + command: start + content: | + [Unit] + Description=Download manifests + Requires=network-online.target + After=network-online.target + Requires=kube-env.service + After=kube-env.service + [Service] + Type=oneshot + RemainAfterExit=yes + EnvironmentFile=/etc/kube-env + ExecStartPre=/usr/bin/mkdir -p /opt/downloads + ExecStartPre=/usr/bin/curl --location --create-dirs --output /opt/downloads/kubernetes-manifests.tar.gz ${KUBE_MANIFESTS_TAR_URL} + # TODO(yifan): Check hash. + ExecStartPre=/usr/bin/mkdir -p /opt/kube-manifests + ExecStart=/usr/bin/tar xf /opt/downloads/kubernetes-manifests.tar.gz -C /opt/kube-manifests --overwrite + + - name: kubernetes-install-node.service + command: start + content: | + [Unit] + Description=Install Kubernetes Server + Requires=network-online.target + After=network-online.target + Requires=kube-env.service + After=kube-env.service + [Service] + Type=oneshot + RemainAfterExit=yes + EnvironmentFile=/etc/kube-env + ExecStartPre=/usr/bin/mkdir -p /opt/downloads + ExecStartPre=/usr/bin/curl --location --create-dirs --output /opt/downloads/kubernetes-server-linux-amd64.tar.gz ${SERVER_BINARY_TAR_URL} + # TODO(yifan): Check hash. + ExecStart=/usr/bin/tar xf /opt/downloads/kubernetes-server-linux-amd64.tar.gz -C /opt --overwrite + + - name: kubelet.service + command: start + content: | + [Unit] + Description=Run Kubelet service + Requires=network-online.target + After=network-online.target + Requires=kube-env.service + After=kube-env.service + Requires=kubernetes-download-manifests.service + After=kubernetes-download-manifests.service + [Service] + EnvironmentFile=/etc/kube-env + ExecStartPre=/usr/bin/curl --fail --silent --show-error \ + -H "X-Google-Metadata-Request: True" \ + -o /run/configure-kubelet.sh \ + http://metadata.google.internal/computeMetadata/v1/instance/attributes/configure-kubelet + ExecStartPre=/usr/bin/chmod 0755 /run/configure-kubelet.sh + ExecStartPre=/run/configure-kubelet.sh + ExecStart=/opt/kubernetes/server/bin/kubelet \ + --api-servers=https://${INSTANCE_PREFIX}-master \ + --enable-debugging-handlers=false \ + --cloud-provider=gce \ + --config=/etc/kubernetes/manifests \ + --allow-privileged=true \ + --v=2 \ + --cluster-dns=${DNS_SERVER_IP} \ + --cluster-domain=${DNS_DOMAIN} \ + --logtostderr=true \ + --container-runtime=${KUBERNETES_CONTAINER_RUNTIME} \ + --rkt-path=/opt/rkt-v${RKT_VERSON}/rkt \ + --configure-cbr0=${KUBERNETES_CONFIGURE_CBR0} \ + --pod-cidr=${MASTER_IP_RANGE} \ + --register-schedulable=false \ + --reconcile-cidr=false + Restart=always + RestartSec=10 + + - name: docker.service + command: start + drop-ins: + - name: 50-docker-opts.conf + content: | + [Service] + Environment='DOCKER_OPTS=--bridge=cbr0 --iptables=false --ip-masq=false' + MountFlags=slave + LimitNOFILE=1048576 + LimitNPROC=1048576 + LimitCORE=infinity + Restart=always + RestartSec=2s + StartLimitInterval=0 + + - name: kubernetes-configure-node.service + command: start + content: | + [Unit] + Description=Configure Node For Kubernetes service + Requires=kubernetes-install-node.service + After=kubernetes-install-node.service + Requires=kubernetes-install-rkt.service + After=kubernetes-install-rkt.service + Requires=kubernetes-download-salt.service + After=kubernetes-download-salt.service + Requires=kubernetes-download-manifests.service + After=kubernetes-download-manifests.service + # Need the kubelet/docker running because we will use docker load for docker images. + Requires=kubelet.service + After=kubelet.service + [Service] + Type=oneshot + RemainAfterExit=yes + EnvironmentFile=/etc/kube-env + ExecStartPre=/usr/bin/curl --fail --silent --show-error \ + -H "X-Google-Metadata-Request: True" \ + -o /run/configure-node.sh \ + http://metadata.google.internal/computeMetadata/v1/instance/attributes/configure-node + ExecStartPre=/usr/bin/chmod 0755 /run/configure-node.sh + ExecStart=/run/configure-node.sh + + - name: kubernetes-addons.service + command: start + content: | + [Unit] + Description=Start Kubernetes addons and watch for updates. + Requires=kubernetes-configure-node.service + After=kubernetes-configure-node.service + [Service] + Environment=KUBECTL_BIN=/opt/kubernetes/server/bin/kubectl + Environment=kubelet_kubeconfig_file=/var/lib/kubelet/kubeconfig + ExecStartPre=/usr/bin/chmod 0755 /opt/kubernetes/saltbase/salt/kube-addons/kube-addons.sh + ExecStart=/opt/kubernetes/saltbase/salt/kube-addons/kube-addons.sh + Restart=always + RestartSec=10 diff --git a/cluster/gce/coreos/node.yaml b/cluster/gce/coreos/node.yaml index 3ca473be617..d2faa576ecd 100644 --- a/cluster/gce/coreos/node.yaml +++ b/cluster/gce/coreos/node.yaml @@ -1,68 +1,5 @@ #cloud-config -write_files: - - path: /run/configure-hostname.sh - permissions: "0755" - content: | - #!/bin/bash -e - set -x - source /etc/kube-env - - hostnamectl set-hostname $(hostname | cut -f1 -d.) - - path: /run/setup-auth.sh - permissions: "0755" - content: | - #!/bin/bash -e - set -x - source /etc/kube-env - - /usr/bin/mkdir -p /var/lib/kubelet - cat > /var/lib/kubelet/kubeconfig << EOF - apiVersion: v1 - kind: Config - users: - - name: kubelet - user: - token: ${KUBELET_TOKEN} - clusters: - - name: local - cluster: - insecure-skip-tls-verify: true - contexts: - - context: - cluster: local - user: kubelet - name: service-account-context - current-context: service-account-context - EOF - - - path: /run/config-kube-proxy.sh - permissions: "0755" - content: | - #!/bin/bash -e - set -x - source /etc/kube-env - - /usr/bin/mkdir -p /var/lib/kube-proxy - cat > /var/lib/kube-proxy/kubeconfig << EOF - apiVersion: v1 - kind: Config - users: - - name: kube-proxy - user: - token: ${KUBE_PROXY_TOKEN} - clusters: - - name: local - cluster: - insecure-skip-tls-verify: true - contexts: - - context: - cluster: local - user: kube-proxy - name: service-account-context - current-context: service-account-context - EOF - coreos: units: - name: kube-env.service @@ -80,30 +17,46 @@ coreos: -o /etc/kube-env.yaml \ http://metadata.google.internal/computeMetadata/v1/instance/attributes/kube-env # Transform the yaml to env file. - ExecStartPre=/usr/bin/cp /etc/kube-env.yaml /etc/kube-env + ExecStartPre=/usr/bin/mv /etc/kube-env.yaml /etc/kube-env ExecStart=/usr/bin/sed -i "s/: '/=/;s/'$//" /etc/kube-env - name: kubernetes-install-rkt.service command: start content: | [Unit] - Description=Fetch Rocket + Description=Fetch rkt Documentation=http://github.com/coreos/rkt Requires=network-online.target After=network-online.target [Service] Type=oneshot + RemainAfterExit=yes EnvironmentFile=/etc/kube-env - ExecStartPre=/usr/bin/rm -rf /opt/rkt - ExecStartPre=/usr/bin/mkdir -p /opt/rkt ExecStartPre=/usr/bin/mkdir -p /etc/rkt - ExecStartPre=/usr/bin/wget \ - -O /opt/rkt/rkt-v${RKT_VERSION}.tar.gz \ - https://github.com/coreos/rkt/releases/download/v${RKT_VERSION}/rkt-v${RKT_VERSION}.tar.gz - ExecStartPre=/usr/bin/tar xzvf /opt/rkt/rkt-v${RKT_VERSION}.tar.gz -C /opt --overwrite - ExecStart=/usr/bin/mv /opt/rkt-v${RKT_VERSION} /opt/rkt/rkt + ExecStartPre=/usr/bin/mkdir -p /opt/downloads + ExecStartPre=/usr/bin/curl --location --create-dirs --output /opt/downloads/rkt.tar.gz https://github.com/coreos/rkt/releases/download/v${RKT_VERSION}/rkt-v${RKT_VERSION}.tar.gz + ExecStart=/usr/bin/tar xf /opt/downloads/rkt.tar.gz -C /opt --overwrite - - name: kubernetes-install-minion.service + - name: kubernetes-download-manifests.service + command: start + content: | + [Unit] + Description=Download manifests + Requires=network-online.target + After=network-online.target + Requires=kube-env.service + After=kube-env.service + [Service] + Type=oneshot + RemainAfterExit=yes + EnvironmentFile=/etc/kube-env + ExecStartPre=/usr/bin/mkdir -p /opt/downloads + ExecStartPre=/usr/bin/curl --location --create-dirs --output /opt/downloads/kubernetes-manifests.tar.gz ${KUBE_MANIFESTS_TAR_URL} + # TODO(yifan): Check hash. + ExecStartPre=/usr/bin/mkdir -p /opt/kube-manifests + ExecStart=/usr/bin/tar xf /opt/downloads/kubernetes-manifests.tar.gz -C /opt/kube-manifests --overwrite + + - name: kubernetes-install-node.service command: start content: | [Unit] @@ -120,33 +73,25 @@ coreos: ExecStartPre=/usr/bin/curl --location --create-dirs --output /opt/kubernetes/pkg/kubernetes-server-linux-amd64.tar.gz ${SERVER_BINARY_TAR_URL} ExecStart=/usr/bin/tar xf /opt/kubernetes/pkg/kubernetes-server-linux-amd64.tar.gz -C /opt --overwrite - - name: kubernetes-preparation.service - command: start - content: | - [Unit] - Description=Configure Node For Kubernetes service - Requires=kubernetes-install-minion.service - After=kubernetes-install-minion.service - Requires=kubernetes-install-rkt.service - After=kubernetes-install-rkt.service - [Service] - Type=oneshot - RemainAfterExit=yes - EnvironmentFile=/etc/kube-env - # TODO(dawnchen): Push this to separate write-files - ExecStart=/run/configure-hostname.sh - - name: kubelet.service command: start content: | [Unit] Description=Run Kubelet service - Requires=kubernetes-preparation.service - After=kubernetes-preparation.service + Requires=network-online.target + After=network-online.target + Requires=kube-env.service + After=kube-env.service + Requires=kubernetes-download-manifests.service + After=kubernetes-download-manifests.service [Service] EnvironmentFile=/etc/kube-env - ExecStartPre=/usr/bin/mkdir -p /etc/kubernetes/manifests - ExecStartPre=/run/setup-auth.sh + ExecStartPre=/usr/bin/curl --fail --silent --show-error \ + -H "X-Google-Metadata-Request: True" \ + -o /run/configure-kubelet.sh \ + http://metadata.google.internal/computeMetadata/v1/instance/attributes/configure-kubelet + ExecStartPre=/usr/bin/chmod 0755 /run/configure-kubelet.sh + ExecStartPre=/run/configure-kubelet.sh ExecStart=/opt/kubernetes/server/bin/kubelet \ --api-servers=https://${INSTANCE_PREFIX}-master \ --enable-debugging-handlers=true \ @@ -158,9 +103,8 @@ coreos: --cluster-domain=${DNS_DOMAIN} \ --logtostderr=true \ --container-runtime=${KUBERNETES_CONTAINER_RUNTIME} \ - --rkt-path=/opt/rkt/rkt/rkt \ - --configure-cbr0=${KUBERNETES_CONFIGURE_CBR0} \ - --pod-cidr=${MASTER_IP_RANGE} \ + --rkt-path=/opt/rkt-v${RKT_VERSON}/rkt \ + --configure-cbr0=${KUBERNETES_CONFIGURE_CBR0} Restart=always RestartSec=10 @@ -169,15 +113,12 @@ coreos: content: | [Unit] Description=Start Kube-proxy service as Daemon - Requires=kubernetes-install-minion.service - After=kubernetes-install-minion.service - Requires=kubernetes-install-rkt.service - After=kubernetes-install-rkt.service + Requires=kubernetes-configure-node.service + After=kubernetes-configure-node.service [Service] EnvironmentFile=/etc/kube-env - ExecStartPre=/run/config-kube-proxy.sh ExecStart=/opt/kubernetes/server/bin/kube-proxy \ - --master=https://${KUBERNETES_MASTER_NAME}.c.${PROJECT_ID}.internal \ + --master=https://${KUBERNETES_MASTER_NAME} \ --kubeconfig=/var/lib/kube-proxy/kubeconfig \ --v=2 \ --logtostderr=true @@ -191,3 +132,32 @@ coreos: content: | [Service] Environment='DOCKER_OPTS=--bridge=cbr0 --iptables=false --ip-masq=false' + MountFlags=slave + LimitNOFILE=1048576 + LimitNPROC=1048576 + LimitCORE=infinity + Restart=always + RestartSec=2s + StartLimitInterval=0 + + - name: kubernetes-configure-node.service + command: start + content: | + [Unit] + Description=Configure Node For Kubernetes service + Requires=kubernetes-install-node.service + After=kubernetes-install-node.service + Requires=kubernetes-install-rkt.service + After=kubernetes-install-rkt.service + Requires=kubernetes-download-manifests.service + After=kubernetes-download-manifests.service + [Service] + Type=oneshot + RemainAfterExit=yes + EnvironmentFile=/etc/kube-env + ExecStartPre=/usr/bin/curl --fail --silent --show-error \ + -H "X-Google-Metadata-Request: True" \ + -o /run/configure-node.sh \ + http://metadata.google.internal/computeMetadata/v1/instance/attributes/configure-node + ExecStartPre=/usr/bin/chmod 0755 /run/configure-node.sh + ExecStart=/run/configure-node.sh