diff --git a/cluster/gce/coreos/configure-node.sh b/cluster/gce/coreos/configure-node.sh old mode 100644 new mode 100755 index c03a2b083da..d462e50ba14 --- a/cluster/gce/coreos/configure-node.sh +++ b/cluster/gce/coreos/configure-node.sh @@ -119,7 +119,7 @@ function wait-url-up() { done } -# Configure addon yamls, and run salt/kube-addons/kube-addon.sh +# Configure addon yamls, and run salt/kube-addons/kube-addons.sh function configure-master-addons() { echo "Configuring master addons" @@ -160,6 +160,10 @@ function configure-master-addons() { CLUSTER_REGISTRY_DISK_SIZE=$(convert-bytes-gce-kube "${CLUSTER_REGISTRY_DISK_SIZE}") evaluate-manifests-dir ${MANIFESTS_DIR}/addons/registry ${addon_dir}/registry fi + + if [[ "${ENABLE_NODE_PROBLEM_DETECTOR}" == "true" ]]; then + evaluate-manifests-dir ${MANIFESTS_DIR}/addons/node-problem-detector ${addon_dir}/node-problem-detector + fi } function configure-master-components() { @@ -169,8 +173,8 @@ function configure-master-components() { configure-kube-apiserver configure-kube-scheduler configure-kube-controller-manager - configure-addon-manager configure-master-addons + configure-addon-manager } # TODO(yifan): Merge this with mount-master-pd() in configure-vm.sh @@ -297,9 +301,10 @@ function load-docker-images() { } -# TODO(yifan): Making this function more generic for other runtimes. function load-master-components-images() { echo "Loading docker images for master components" + export RKT_BIN=/opt/rkt/rkt + export DOCKER2ACI_BIN=/opt/docker2aci/docker2aci ${SALT_DIR}/install.sh ${KUBE_BIN_TAR} ${SALT_DIR}/salt/kube-master-addons/kube-master-addons.sh diff --git a/cluster/gce/coreos/helper.sh b/cluster/gce/coreos/helper.sh index 275f1e764fc..6e0fa89b298 100755 --- a/cluster/gce/coreos/helper.sh +++ b/cluster/gce/coreos/helper.sh @@ -23,7 +23,7 @@ function create-node-instance-template() { local template_name="$1" create-node-template "$template_name" "${scope_flags}" \ "kube-env=${KUBE_TEMP}/node-kube-env.yaml" \ - "user-data=${KUBE_ROOT}/cluster/gce/coreos/node.yaml" \ + "user-data=${KUBE_ROOT}/cluster/gce/coreos/node-${CONTAINER_RUNTIME}.yaml" \ "configure-node=${KUBE_ROOT}/cluster/gce/coreos/configure-node.sh" \ "configure-kubelet=${KUBE_ROOT}/cluster/gce/coreos/configure-kubelet.sh" \ "cluster-name=${KUBE_TEMP}/cluster-name.txt" @@ -65,7 +65,7 @@ function create-master-instance() { --scopes "storage-ro,compute-rw,monitoring,logging-write" \ --can-ip-forward \ --metadata-from-file \ - "kube-env=${KUBE_TEMP}/master-kube-env.yaml,user-data=${KUBE_ROOT}/cluster/gce/coreos/master.yaml,configure-node=${KUBE_ROOT}/cluster/gce/coreos/configure-node.sh,configure-kubelet=${KUBE_ROOT}/cluster/gce/coreos/configure-kubelet.sh,cluster-name=${KUBE_TEMP}/cluster-name.txt" \ + "kube-env=${KUBE_TEMP}/master-kube-env.yaml,user-data=${KUBE_ROOT}/cluster/gce/coreos/master-${CONTAINER_RUNTIME}.yaml,configure-node=${KUBE_ROOT}/cluster/gce/coreos/configure-node.sh,configure-kubelet=${KUBE_ROOT}/cluster/gce/coreos/configure-kubelet.sh,cluster-name=${KUBE_TEMP}/cluster-name.txt" \ --disk "name=${MASTER_NAME}-pd,device-name=master-pd,mode=rw,boot=no,auto-delete=no" \ --boot-disk-size "${MASTER_ROOT_DISK_SIZE:-10}" \ ${preemptible_master} diff --git a/cluster/gce/coreos/kube-manifests/addons/dns/skydns-rc.yaml b/cluster/gce/coreos/kube-manifests/addons/dns/skydns-rc.yaml index ff3e6e08795..1a1ee3b1c0f 100644 --- a/cluster/gce/coreos/kube-manifests/addons/dns/skydns-rc.yaml +++ b/cluster/gce/coreos/kube-manifests/addons/dns/skydns-rc.yaml @@ -1,27 +1,27 @@ apiVersion: v1 kind: ReplicationController metadata: - name: kube-dns-v11 + name: kube-dns-v14 namespace: kube-system labels: k8s-app: kube-dns - version: v11 + version: v14 kubernetes.io/cluster-service: "true" spec: replicas: ${DNS_REPLICAS} selector: k8s-app: kube-dns - version: v11 + version: v14 template: metadata: labels: k8s-app: kube-dns - version: v11 + version: v14 kubernetes.io/cluster-service: "true" spec: containers: - - name: etcd - image: gcr.io/google_containers/etcd-amd64:2.2.1 + - name: kubedns + image: gcr.io/google_containers/kubedns-amd64:1.3 resources: # TODO: Set memory limits when we've profiled the container for large # clusters, then set request = limit to keep this container in @@ -29,33 +29,6 @@ spec: # "burstable" category so the kubelet doesn't backoff from restarting it. limits: cpu: 100m - memory: 500Mi - requests: - cpu: 100m - memory: 50Mi - command: - - /usr/local/bin/etcd - - -data-dir - - /var/etcd/data - - -listen-client-urls - - http://127.0.0.1:2379,http://127.0.0.1:4001 - - -advertise-client-urls - - http://127.0.0.1:2379,http://127.0.0.1:4001 - - -initial-cluster-token - - skydns-etcd - volumeMounts: - - name: etcd-storage - mountPath: /var/etcd/data - - name: kube2sky - image: gcr.io/google_containers/kube2sky:1.15 - resources: - # TODO: Set memory limits when we've profiled the container for large - # clusters, then set request = limit to keep this container in - # guaranteed class. Currently, this container falls into the - # "burstable" category so the kubelet doesn't backoff from restarting it. - limits: - cpu: 100m - # Kube2sky watches all pods. memory: 200Mi requests: cpu: 100m @@ -78,32 +51,23 @@ spec: # only setup the /readiness HTTP server once that's available. initialDelaySeconds: 30 timeoutSeconds: 5 - command: - - /kube2sky args: - # command = "/kube2sky" - - --domain=${DNS_DOMAIN} - - name: skydns - image: gcr.io/google_containers/skydns:2015-10-13-8c72f8c - resources: - # TODO: Set memory limits when we've profiled the container for large - # clusters, then set request = limit to keep this container in - # guaranteed class. Currently, this container falls into the - # "burstable" category so the kubelet doesn't backoff from restarting it. - limits: - cpu: 100m - memory: 200Mi - requests: - cpu: 100m - memory: 50Mi - command: - - /skydns + # command = "/kube-dns" + - --domain=${DNS_DOMAIN}. + - --dns-port=10053 + ports: + - containerPort: 10053 + name: dns-local + protocol: UDP + - containerPort: 10053 + name: dns-tcp-local + protocol: TCP + - name: dnsmasq + image: gcr.io/google_containers/dnsmasq:1.1 args: - # command = "/skydns" - - -machines=http://127.0.0.1:4001 - - -addr=0.0.0.0:53 - - -ns-rotate=false - - -domain=${DNS_DOMAIN}. + - --cache-size=1000 + - --no-resolv + - --server=127.0.0.1#10053 ports: - containerPort: 53 name: dns @@ -112,7 +76,7 @@ spec: name: dns-tcp protocol: TCP - name: healthz - image: gcr.io/google_containers/exechealthz:1.0 + image: gcr.io/google_containers/exechealthz-amd64:1.0 resources: # keep request = limit to keep this container in guaranteed class limits: @@ -121,15 +85,10 @@ spec: requests: cpu: 10m memory: 20Mi - command: - - /exechealthz args: - -cmd=nslookup kubernetes.default.svc.${DNS_DOMAIN} 127.0.0.1 >/dev/null - -port=8080 ports: - containerPort: 8080 protocol: TCP - volumes: - - name: etcd-storage - emptyDir: {} - dnsPolicy: Default # Don't use cluster DNS. \ No newline at end of file + dnsPolicy: Default # Don't use cluster DNS. diff --git a/cluster/gce/coreos/kube-manifests/addons/fluentd-elasticsearch/es-controller.yaml b/cluster/gce/coreos/kube-manifests/addons/fluentd-elasticsearch/es-controller.yaml index 509c3456751..c875984e108 100644 --- a/cluster/gce/coreos/kube-manifests/addons/fluentd-elasticsearch/es-controller.yaml +++ b/cluster/gce/coreos/kube-manifests/addons/fluentd-elasticsearch/es-controller.yaml @@ -20,7 +20,7 @@ spec: kubernetes.io/cluster-service: "true" spec: containers: - - image: gcr.io/google_containers/elasticsearch:1.8 + - image: gcr.io/google_containers/elasticsearch:1.9 name: elasticsearch-logging resources: # keep request = limit to keep this container in guaranteed class @@ -40,4 +40,4 @@ spec: mountPath: /data volumes: - name: es-persistent-storage - emptyDir: {} \ No newline at end of file + emptyDir: {} diff --git a/cluster/gce/coreos/kube-manifests/addons/node-problem-detector/node-problem-detector.yaml b/cluster/gce/coreos/kube-manifests/addons/node-problem-detector/node-problem-detector.yaml new file mode 100644 index 00000000000..e206cdf2092 --- /dev/null +++ b/cluster/gce/coreos/kube-manifests/addons/node-problem-detector/node-problem-detector.yaml @@ -0,0 +1,44 @@ +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: node-problem-detector-v0.1 + namespace: kube-system + labels: + k8s-app: node-problem-detector + version: v0.1 + kubernetes.io/cluster-service: "true" +spec: + template: + metadata: + labels: + k8s-app: node-problem-detector + version: v0.1 + kubernetes.io/cluster-service: "true" + spec: + hostNetwork: true + containers: + - name: node-problem-detector + image: gcr.io/google_containers/node-problem-detector:v0.1 + env: + # Config the host ip and port of apiserver. + - name: "KUBERNETES_SERVICE_HOST" + value: ${INSTANCE_PREFIX}-master + - name: "KUBERNETES_SERVICE_PORT" + value: "443" + securityContext: + privileged: true + resources: + limits: + cpu: "200m" + memory: "100Mi" + requests: + cpu: "20m" + memory: "20Mi" + volumeMounts: + - name: log + mountPath: /log + readOnly: true + volumes: + - name: log + hostPath: + path: /var/log/ diff --git a/cluster/gce/coreos/kube-manifests/etcd-events.yaml b/cluster/gce/coreos/kube-manifests/etcd-events.yaml index d6a92f0c62e..b682ec9aa65 100644 --- a/cluster/gce/coreos/kube-manifests/etcd-events.yaml +++ b/cluster/gce/coreos/kube-manifests/etcd-events.yaml @@ -14,7 +14,7 @@ spec: --bind-addr=127.0.0.1:4002 --data-dir=/var/etcd/data-events 1>>/var/log/etcd-events.log 2>&1 - image: gcr.io/google_containers/etcd:2.0.12 + image: gcr.io/google_containers/etcd:2.2.1 imagePullPolicy: IfNotPresent livenessProbe: httpGet: diff --git a/cluster/gce/coreos/kube-manifests/kube-addon-manager.yaml b/cluster/gce/coreos/kube-manifests/kube-addon-manager.yaml index d46b9888117..e63f23b5f38 100644 --- a/cluster/gce/coreos/kube-manifests/kube-addon-manager.yaml +++ b/cluster/gce/coreos/kube-manifests/kube-addon-manager.yaml @@ -12,7 +12,9 @@ spec: resources: requests: cpu: 5m - memory: 50Mi + # TODO(yifan): Figure out what's the memory usage should be here. + # See https://github.com/kubernetes/kubernetes/issues/23641. + memory: 100Mi volumeMounts: - mountPath: /etc/kubernetes/ name: addons diff --git a/cluster/gce/coreos/master.yaml b/cluster/gce/coreos/master-docker.yaml similarity index 86% rename from cluster/gce/coreos/master.yaml rename to cluster/gce/coreos/master-docker.yaml index 11a83fd094f..67240950185 100644 --- a/cluster/gce/coreos/master.yaml +++ b/cluster/gce/coreos/master-docker.yaml @@ -35,27 +35,9 @@ coreos: Type=oneshot RemainAfterExit=yes ExecStartPre=/usr/bin/mkdir -p /opt/cni - ExecStartPre=/usr/bin/curl --fail --silent --show-error --location --create-dirs --output /opt/downloads/cni.tar.gz https://storage.googleapis.com/kubernetes-release/network-plugins/cni-42c4cb842dad606a84e93aad5a4484ded48e3046.tar.gz + ExecStartPre=/usr/bin/curl --fail --silent --show-error --location --create-dirs --output /opt/downloads/cni.tar.gz https://storage.googleapis.com/kubernetes-release/network-plugins/cni-c864f0e1ea73719b8f4582402b0847064f9883b0.tar.gz ExecStart=/usr/bin/tar xf /opt/downloads/cni.tar.gz -C /opt/cni/ - - name: kubernetes-install-rkt.service - command: start - content: | - [Unit] - Description=Fetch rkt - Documentation=http://github.com/coreos/rkt - Requires=network-online.target - After=network-online.target - Requires=kube-env.service - After=kube-env.service - [Service] - Type=oneshot - RemainAfterExit=yes - EnvironmentFile=/etc/kube-env - ExecStartPre=/usr/bin/mkdir -p /etc/rkt /opt/downloads /opt/rkt/ - ExecStartPre=/usr/bin/curl --fail --silent --location --create-dirs --output /opt/downloads/rkt.tar.gz https://github.com/coreos/rkt/releases/download/v${RKT_VERSION}/rkt-v${RKT_VERSION}.tar.gz - ExecStart=/usr/bin/tar --strip-components=1 -xf /opt/downloads/rkt.tar.gz -C /opt/rkt/ --overwrite - - name: kubernetes-download-salt.service command: start content: | @@ -139,7 +121,7 @@ coreos: --cluster-dns=${DNS_SERVER_IP} \ --cluster-domain=${DNS_DOMAIN} \ --logtostderr=true \ - --container-runtime=${KUBERNETES_CONTAINER_RUNTIME} \ + --container-runtime=docker \ --rkt-path=/opt/rkt/rkt \ --rkt-stage1-image=/opt/rkt/stage1-coreos.aci \ --configure-cbr0=${KUBERNETES_CONFIGURE_CBR0} \ @@ -149,7 +131,7 @@ coreos: Restart=always RestartSec=10 KillMode=process - + - name: docker.service drop-ins: - name: 50-docker-opts.conf @@ -171,8 +153,6 @@ coreos: Description=Configure Node For Kubernetes service Requires=kubernetes-install-node.service After=kubernetes-install-node.service - Requires=kubernetes-install-rkt.service - After=kubernetes-install-rkt.service Requires=kubernetes-download-salt.service After=kubernetes-download-salt.service Requires=kubernetes-download-manifests.service diff --git a/cluster/gce/coreos/master-rkt.yaml b/cluster/gce/coreos/master-rkt.yaml new file mode 100644 index 00000000000..1f7060568c5 --- /dev/null +++ b/cluster/gce/coreos/master-rkt.yaml @@ -0,0 +1,209 @@ +#cloud-config + +coreos: + update: + reboot-strategy: off + units: + - name: locksmithd.service + mask: true + - name: kube-env.service + command: start + content: | + [Unit] + Description=Fetch kubernetes-node-environment + Requires=network-online.target + After=network-online.target + [Service] + Type=oneshot + RemainAfterExit=yes + ExecStartPre=/usr/bin/curl --fail --silent --show-error \ + -H "X-Google-Metadata-Request: True" \ + -o /etc/kube-env.yaml \ + http://metadata.google.internal/computeMetadata/v1/instance/attributes/kube-env + # Transform the yaml to env file. + ExecStartPre=/usr/bin/mv /etc/kube-env.yaml /etc/kube-env + ExecStart=/usr/bin/sed -i "s/: '/=/;s/'$//" /etc/kube-env + + - name: kubernetes-install-cni.service + command: start + content: | + [Unit] + Description=Download cni + Requires=network-online.target + After=network-online.target + [Service] + Type=oneshot + RemainAfterExit=yes + ExecStartPre=/usr/bin/mkdir -p /opt/cni + ExecStartPre=/usr/bin/curl --fail --silent --show-error --location --create-dirs --output /opt/downloads/cni.tar.gz https://storage.googleapis.com/kubernetes-release/network-plugins/cni-c864f0e1ea73719b8f4582402b0847064f9883b0.tar.gz + ExecStart=/usr/bin/tar xf /opt/downloads/cni.tar.gz -C /opt/cni/ + + - name: kubernetes-install-docker2aci.service + command: start + content: | + [Unit] + Description=Download docker2aci + Requires=network-online.target + After=network-online.target + [Service] + Type=oneshot + RemainAfterExit=yes + ExecStartPre=/usr/bin/mkdir -p /opt/docker2aci + ExecStartPre=/usr/bin/curl --fail --silent --show-error --location --create-dirs --output /opt/downloads/docker2aci.tar.gz https://github.com/appc/docker2aci/releases/download/v0.11.1/docker2aci-v0.11.1.tar.gz + ExecStart=/usr/bin/tar --strip-components=1 -xf /opt/downloads/docker2aci.tar.gz -C /opt/docker2aci/ --overwrite + + - name: kubernetes-install-rkt.service + command: start + content: | + [Unit] + Description=Fetch rkt + Documentation=http://github.com/coreos/rkt + Requires=network-online.target + After=network-online.target + Requires=kube-env.service + After=kube-env.service + [Service] + Type=oneshot + RemainAfterExit=yes + EnvironmentFile=/etc/kube-env + ExecStartPre=/usr/bin/mkdir -p /etc/rkt /opt/downloads /opt/rkt/ + ExecStartPre=/usr/bin/curl --fail --silent --location --create-dirs --output /opt/downloads/rkt.tar.gz https://github.com/coreos/rkt/releases/download/v${RKT_VERSION}/rkt-v${RKT_VERSION}.tar.gz + ExecStart=/usr/bin/tar --strip-components=1 -xf /opt/downloads/rkt.tar.gz -C /opt/rkt/ --overwrite + + - name: kubernetes-download-salt.service + command: start + content: | + [Unit] + Description=Download salt + Requires=network-online.target + After=network-online.target + Requires=kube-env.service + After=kube-env.service + [Service] + Type=oneshot + RemainAfterExit=yes + EnvironmentFile=/etc/kube-env + ExecStartPre=/usr/bin/mkdir -p /opt/downloads + ExecStartPre=/usr/bin/curl --fail --silent --show-error --location --create-dirs --output \ + /opt/downloads/kubernetes-salt.tar.gz ${SALT_TAR_URL} + # TODO(yifan): Check hash. + ExecStart=/usr/bin/tar xf /opt/downloads/kubernetes-salt.tar.gz -C /opt --overwrite + + - name: kubernetes-download-manifests.service + command: start + content: | + [Unit] + Description=Download manifests + Requires=network-online.target + After=network-online.target + Requires=kube-env.service + After=kube-env.service + [Service] + Type=oneshot + RemainAfterExit=yes + EnvironmentFile=/etc/kube-env + ExecStartPre=/usr/bin/mkdir -p /opt/downloads + ExecStartPre=/usr/bin/curl --fail --silent --show-error --location --create-dirs --output \ + /opt/downloads/kubernetes-manifests.tar.gz ${KUBE_MANIFESTS_TAR_URL} + # TODO(yifan): Check hash. + ExecStartPre=/usr/bin/mkdir -p /opt/kube-manifests + ExecStart=/usr/bin/tar xf /opt/downloads/kubernetes-manifests.tar.gz -C /opt/kube-manifests --overwrite + + - name: kubernetes-install-node.service + command: start + content: | + [Unit] + Description=Install Kubernetes Server + Requires=network-online.target + After=network-online.target + Requires=kube-env.service + After=kube-env.service + [Service] + Type=oneshot + RemainAfterExit=yes + EnvironmentFile=/etc/kube-env + ExecStartPre=/usr/bin/mkdir -p /opt/downloads + ExecStartPre=/usr/bin/curl --fail --silent --show-error --location --create-dirs --output \ + /opt/downloads/kubernetes-server-linux-amd64.tar.gz ${SERVER_BINARY_TAR_URL} + # TODO(yifan): Check hash. + ExecStart=/usr/bin/tar xf /opt/downloads/kubernetes-server-linux-amd64.tar.gz -C /opt --overwrite + + - name: rkt-api-service.service + command: start + content: | + [Unit] + Description=Start rkt API service as Daemon + Requires=kubernetes-install-rkt.service + After=kubernetes-install-rkt.service + [Service] + ExecStart=/opt/rkt/rkt api-service + Restart=always + RestartSec=10 + + - name: kubelet.service + command: start + content: | + [Unit] + Description=Run Kubelet service + Requires=network-online.target kube-env.service kubernetes-download-manifests.service kubernetes-install-cni.service + After=network-online.target kube-env.service kubernetes-download-manifests.service kubernetes-install-cni.service + [Service] + EnvironmentFile=/etc/kube-env + ExecStartPre=/usr/bin/curl --fail --silent --show-error \ + -H "X-Google-Metadata-Request: True" \ + -o /run/configure-kubelet.sh \ + http://metadata.google.internal/computeMetadata/v1/instance/attributes/configure-kubelet + ExecStartPre=/usr/bin/chmod 0755 /run/configure-kubelet.sh + ExecStartPre=/run/configure-kubelet.sh + ExecStart=/opt/kubernetes/server/bin/kubelet \ + --api-servers=https://${INSTANCE_PREFIX}-master \ + --enable-debugging-handlers=false \ + --cloud-provider=gce \ + --config=/etc/kubernetes/manifests \ + --allow-privileged=true \ + --v=2 \ + --cluster-dns=${DNS_SERVER_IP} \ + --cluster-domain=${DNS_DOMAIN} \ + --logtostderr=true \ + --container-runtime=rkt \ + --rkt-path=/opt/rkt/rkt \ + --rkt-stage1-image=/opt/rkt/stage1-coreos.aci \ + --configure-cbr0=${KUBERNETES_CONFIGURE_CBR0} \ + --pod-cidr=${MASTER_IP_RANGE} \ + --register-schedulable=false \ + --reconcile-cidr=false + Restart=always + RestartSec=10 + KillMode=process + + - name: docker.service + command: stop + + - name: kubernetes-configure-node.service + command: start + content: | + [Unit] + Description=Configure Node For Kubernetes service + Requires=kubernetes-install-node.service + After=kubernetes-install-node.service + Requires=kubernetes-install-rkt.service + After=kubernetes-install-rkt.service + Requires=kubernetes-download-salt.service + After=kubernetes-download-salt.service + Requires=kubernetes-download-manifests.service + After=kubernetes-download-manifests.service + Requires=kubernetes-install-docker2aci.service + After=kubernetes-install-docker2aci.service + # Need the kubelet/docker running because we will use docker load for docker images. + Requires=kubelet.service + After=kubelet.service + [Service] + Type=oneshot + RemainAfterExit=yes + EnvironmentFile=/etc/kube-env + ExecStartPre=/usr/bin/curl --fail --silent --show-error \ + -H "X-Google-Metadata-Request: True" \ + -o /run/configure-node.sh \ + http://metadata.google.internal/computeMetadata/v1/instance/attributes/configure-node + ExecStartPre=/usr/bin/chmod 0755 /run/configure-node.sh + ExecStart=/run/configure-node.sh diff --git a/cluster/gce/coreos/node-docker.yaml b/cluster/gce/coreos/node-docker.yaml new file mode 100644 index 00000000000..5096f9e5300 --- /dev/null +++ b/cluster/gce/coreos/node-docker.yaml @@ -0,0 +1,161 @@ +#cloud-config + +coreos: + update: + reboot-strategy: off + units: + - name: locksmithd.service + mask: true + - name: kube-env.service + command: start + content: | + [Unit] + Description=Fetch kubernetes-node-environment + Requires=network-online.target + After=network-online.target + [Service] + Type=oneshot + RemainAfterExit=yes + ExecStartPre=/usr/bin/curl --fail --silent --show-error \ + -H "X-Google-Metadata-Request: True" \ + -o /etc/kube-env.yaml \ + http://metadata.google.internal/computeMetadata/v1/instance/attributes/kube-env + # Transform the yaml to env file. + ExecStartPre=/usr/bin/mv /etc/kube-env.yaml /etc/kube-env + ExecStart=/usr/bin/sed -i "s/: '/=/;s/'$//" /etc/kube-env + + - name: kubernetes-install-cni.service + command: start + content: | + [Unit] + Description=Download cni + Requires=network-online.target + After=network-online.target + [Service] + Type=oneshot + RemainAfterExit=yes + ExecStartPre=/usr/bin/mkdir -p /opt/cni + ExecStartPre=/usr/bin/curl --fail --silent --show-error --location --create-dirs --output /opt/downloads/cni.tar.gz https://storage.googleapis.com/kubernetes-release/network-plugins/cni-c864f0e1ea73719b8f4582402b0847064f9883b0.tar.gz + ExecStart=/usr/bin/tar xf /opt/downloads/cni.tar.gz -C /opt/cni/ + + - name: kubernetes-download-manifests.service + command: start + content: | + [Unit] + Description=Download manifests + Requires=network-online.target + After=network-online.target + Requires=kube-env.service + After=kube-env.service + [Service] + Type=oneshot + RemainAfterExit=yes + EnvironmentFile=/etc/kube-env + ExecStartPre=/usr/bin/mkdir -p /opt/downloads + ExecStartPre=/usr/bin/curl --fail --silent --show-error --location --create-dirs --output \ + /opt/downloads/kubernetes-manifests.tar.gz ${KUBE_MANIFESTS_TAR_URL} + # TODO(yifan): Check hash. + ExecStartPre=/usr/bin/mkdir -p /opt/kube-manifests + ExecStart=/usr/bin/tar xf /opt/downloads/kubernetes-manifests.tar.gz -C /opt/kube-manifests --overwrite + + - name: kubernetes-install-node.service + command: start + content: | + [Unit] + Description=Install Kubernetes Server + Requires=network-online.target + After=network-online.target + Requires=kube-env.service + After=kube-env.service + [Service] + Type=oneshot + RemainAfterExit=yes + EnvironmentFile=/etc/kube-env + ExecStartPre=/usr/bin/mkdir -p /opt/kubernetes/pkg + ExecStartPre=/usr/bin/curl --fail --silent --show-error --location --create-dirs --output \ + /opt/kubernetes/pkg/kubernetes-server-linux-amd64.tar.gz ${SERVER_BINARY_TAR_URL} + ExecStart=/usr/bin/tar xf /opt/kubernetes/pkg/kubernetes-server-linux-amd64.tar.gz -C /opt --overwrite + + - name: kubelet.service + command: start + content: | + [Unit] + Description=Run Kubelet service + Requires=network-online.target kube-env.service kubernetes-download-manifests.service kubernetes-install-cni.service + After=network-online.target kube-env.service kubernetes-download-manifests.service kubernetes-install-cni.service + [Service] + EnvironmentFile=/etc/kube-env + ExecStartPre=/usr/bin/curl --fail --silent --show-error \ + -H "X-Google-Metadata-Request: True" \ + -o /run/configure-kubelet.sh \ + http://metadata.google.internal/computeMetadata/v1/instance/attributes/configure-kubelet + ExecStartPre=/usr/bin/chmod 0755 /run/configure-kubelet.sh + ExecStartPre=/run/configure-kubelet.sh + ExecStart=/opt/kubernetes/server/bin/kubelet \ + --api-servers=https://${INSTANCE_PREFIX}-master \ + --enable-debugging-handlers=true \ + --cloud-provider=gce \ + --config=/etc/kubernetes/manifests \ + --allow-privileged=true \ + --v=2 \ + --cluster-dns=${DNS_SERVER_IP} \ + --cluster-domain=${DNS_DOMAIN} \ + --logtostderr=true \ + --container-runtime=docker \ + --configure-cbr0=${KUBERNETES_CONFIGURE_CBR0} \ + --network-plugin=${NETWORK_PROVIDER} \ + --reconcile-cidr=true + Restart=always + RestartSec=10 + KillMode=process + + - name: kube-proxy.service + command: start + content: | + [Unit] + Description=Start Kube-proxy service as Daemon + Requires=kubernetes-configure-node.service + After=kubernetes-configure-node.service + [Service] + EnvironmentFile=/etc/kube-env + ExecStart=/opt/kubernetes/server/bin/kube-proxy \ + --master=https://${KUBERNETES_MASTER_NAME} \ + --kubeconfig=/var/lib/kube-proxy/kubeconfig \ + --v=2 \ + --logtostderr=true + Restart=always + RestartSec=10 + + - name: docker.service + drop-ins: + - name: 50-docker-opts.conf + content: | + [Service] + Environment='DOCKER_OPTS= --iptables=false --ip-masq=false' + MountFlags=slave + LimitNOFILE=1048576 + LimitNPROC=1048576 + LimitCORE=infinity + Restart=always + RestartSec=2s + StartLimitInterval=0 + + - name: kubernetes-configure-node.service + command: start + content: | + [Unit] + Description=Configure Node For Kubernetes service + Requires=kubernetes-install-node.service + After=kubernetes-install-node.service + Requires=kubernetes-download-manifests.service + After=kubernetes-download-manifests.service + [Service] + Type=oneshot + RemainAfterExit=yes + EnvironmentFile=/etc/kube-env + ExecStartPre=/usr/bin/curl --fail --silent --show-error \ + -H "X-Google-Metadata-Request: True" \ + -o /run/configure-node.sh \ + http://metadata.google.internal/computeMetadata/v1/instance/attributes/configure-node + ExecStartPre=/usr/bin/chmod 0755 /run/configure-node.sh + ExecStart=/run/configure-node.sh diff --git a/cluster/gce/coreos/node.yaml b/cluster/gce/coreos/node-rkt.yaml similarity index 92% rename from cluster/gce/coreos/node.yaml rename to cluster/gce/coreos/node-rkt.yaml index 4d07b94c6aa..2b72e16cf38 100644 --- a/cluster/gce/coreos/node.yaml +++ b/cluster/gce/coreos/node-rkt.yaml @@ -35,7 +35,7 @@ coreos: Type=oneshot RemainAfterExit=yes ExecStartPre=/usr/bin/mkdir -p /opt/cni - ExecStartPre=/usr/bin/curl --fail --silent --show-error --location --create-dirs --output /opt/downloads/cni.tar.gz https://storage.googleapis.com/kubernetes-release/network-plugins/cni-42c4cb842dad606a84e93aad5a4484ded48e3046.tar.gz + ExecStartPre=/usr/bin/curl --fail --silent --show-error --location --create-dirs --output /opt/downloads/cni.tar.gz https://storage.googleapis.com/kubernetes-release/network-plugins/cni-c864f0e1ea73719b8f4582402b0847064f9883b0.tar.gz ExecStart=/usr/bin/tar xf /opt/downloads/cni.tar.gz -C /opt/cni/ - name: kubernetes-install-rkt.service @@ -131,10 +131,12 @@ coreos: --cluster-dns=${DNS_SERVER_IP} \ --cluster-domain=${DNS_DOMAIN} \ --logtostderr=true \ - --container-runtime=${KUBERNETES_CONTAINER_RUNTIME} \ + --container-runtime=rkt \ --rkt-path=/opt/rkt/rkt \ --rkt-stage1-image=/opt/rkt/stage1-coreos.aci \ - --configure-cbr0=${KUBERNETES_CONFIGURE_CBR0} + --configure-cbr0=${KUBERNETES_CONFIGURE_CBR0} \ + --network-plugin=kubenet \ + --reconcile-cidr=true Restart=always RestartSec=10 KillMode=process @@ -157,18 +159,7 @@ coreos: RestartSec=10 - name: docker.service - drop-ins: - - name: 50-docker-opts.conf - content: | - [Service] - Environment='DOCKER_OPTS=--bridge=cbr0 --iptables=false --ip-masq=false' - MountFlags=slave - LimitNOFILE=1048576 - LimitNPROC=1048576 - LimitCORE=infinity - Restart=always - RestartSec=2s - StartLimitInterval=0 + command: stop - name: kubernetes-configure-node.service command: start diff --git a/cluster/saltbase/salt/kube-master-addons/kube-master-addons.sh b/cluster/saltbase/salt/kube-master-addons/kube-master-addons.sh index 4041e8f21f7..a91ecb77dbd 100755 --- a/cluster/saltbase/salt/kube-master-addons/kube-master-addons.sh +++ b/cluster/saltbase/salt/kube-master-addons/kube-master-addons.sh @@ -15,52 +15,76 @@ # limitations under the License. # loadedImageFlags is a bit-flag to track which docker images loaded successfully. -let loadedImageFlags=0 -while true; do - restart_docker=false - - if which docker 1>/dev/null 2>&1; then - - timeout 30 docker load -i /srv/salt/kube-bins/kube-apiserver.tar 1>/dev/null 2>&1 - rc=$? - if [[ $rc == 0 ]]; then - let loadedImageFlags="$loadedImageFlags|1" - elif [[ $rc == 124 ]]; then - restart_docker=true +function load-docker-images() { + let loadedImageFlags=0 + + while true; do + restart_docker=false + + if which docker 1>/dev/null 2>&1; then + + timeout 30 docker load -i /srv/salt/kube-bins/kube-apiserver.tar 1>/dev/null 2>&1 + rc=$? + if [[ $rc == 0 ]]; then + let loadedImageFlags="$loadedImageFlags|1" + elif [[ $rc == 124 ]]; then + restart_docker=true + fi + + timeout 30 docker load -i /srv/salt/kube-bins/kube-scheduler.tar 1>/dev/null 2>&1 + rc=$? + if [[ $rc == 0 ]]; then + let loadedImageFlags="$loadedImageFlags|2" + elif [[ $rc == 124 ]]; then + restart_docker=true + fi + + timeout 30 docker load -i /srv/salt/kube-bins/kube-controller-manager.tar 1>/dev/null 2>&1 + rc=$? + if [[ $rc == 0 ]]; then + let loadedImageFlags="$loadedImageFlags|4" + elif [[ $rc == 124 ]]; then + restart_docker=true + fi fi - - timeout 30 docker load -i /srv/salt/kube-bins/kube-scheduler.tar 1>/dev/null 2>&1 - rc=$? - if [[ $rc == 0 ]]; then - let loadedImageFlags="$loadedImageFlags|2" - elif [[ $rc == 124 ]]; then - restart_docker=true + + # required docker images got installed. exit while loop. + if [[ $loadedImageFlags == 7 ]]; then break; fi + + # Sometimes docker load hang, restart docker daemon resolve the issue + if [[ $restart_docker ]]; then + if ! service docker restart; then # Try systemctl if there's no service command. + systemctl restart docker + fi fi + + # sleep for 15 seconds before attempting to load docker images again + sleep 15 + + done +} - timeout 30 docker load -i /srv/salt/kube-bins/kube-controller-manager.tar 1>/dev/null 2>&1 - rc=$? - if [[ $rc == 0 ]]; then - let loadedImageFlags="$loadedImageFlags|4" - elif [[ $rc == 124 ]]; then - restart_docker=true - fi - fi +function convert-rkt-image() { + (cd /tmp; ${DOCKER2ACI_BIN} $1) +} - # required docker images got installed. exit while loop. - if [[ $loadedImageFlags == 7 ]]; then break; fi +function load-rkt-images() { + convert-rkt-image /srv/salt/kube-bins/kube-apiserver.tar + convert-rkt-image /srv/salt/kube-bins/kube-scheduler.tar + convert-rkt-image /srv/salt/kube-bins/kube-controller-manager.tar - # Sometimes docker load hang, restart docker daemon resolve the issue - if [[ $restart_docker ]]; then - if ! service docker restart; then # Try systemctl if there's no service command. - systemctl restart docker - fi - fi + # Currently, we can't run docker image tarballs directly, + # So we use 'rkt fetch' to load the docker images into rkt image stores. + # see https://github.com/coreos/rkt/issues/2392. + ${RKT_BIN} fetch /tmp/*.aci --insecure-options=image +} - # sleep for 15 seconds before attempting to load docker images again - sleep 15 - -done +if [[ "${KUBERNETES_CONTAINER_RUNTIME}" == "rkt" ]]; then + load-rkt-images +else + load-docker-images +fi # Now exit. After kube-push, salt will notice that the service is down and it # will start it and new docker images will be loaded.