From bff87ff2a70c750f461c2320ffd1502dd1ead3e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lucas=20K=C3=A4ldstr=C3=B6m?= Date: Fri, 20 May 2016 19:27:42 +0300 Subject: [PATCH 1/2] Make the addon-manager cross-platform, change naming to binary-arch:version, remove deprecated kubectl command, add support for DaemonSets --- cluster/addons/addon-manager/.gitignore | 1 - cluster/addons/addon-manager/Dockerfile | 10 ++- cluster/addons/addon-manager/Makefile | 64 +++++++++++++++---- cluster/addons/addon-manager/README.md | 37 +++++++++++ .../addons/addon-manager/kube-addon-update.sh | 3 +- 5 files changed, 98 insertions(+), 17 deletions(-) delete mode 100644 cluster/addons/addon-manager/.gitignore create mode 100644 cluster/addons/addon-manager/README.md diff --git a/cluster/addons/addon-manager/.gitignore b/cluster/addons/addon-manager/.gitignore deleted file mode 100644 index 4eb4f5f7b24..00000000000 --- a/cluster/addons/addon-manager/.gitignore +++ /dev/null @@ -1 +0,0 @@ -kubectl diff --git a/cluster/addons/addon-manager/Dockerfile b/cluster/addons/addon-manager/Dockerfile index 09653ab99a8..c33994394eb 100644 --- a/cluster/addons/addon-manager/Dockerfile +++ b/cluster/addons/addon-manager/Dockerfile @@ -12,13 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM python:2.7-slim +FROM BASEIMAGE + +# If we're building for another architecture than amd64, the CROSS_BUILD_ placeholder is removed so e.g. CROSS_BUILD_COPY turns into COPY +# If we're building normally, for amd64, CROSS_BUILD lines are removed +CROSS_BUILD_COPY qemu-ARCH-static /usr/bin/ RUN pip install pyyaml ADD kube-addons.sh /opt/ ADD kube-addon-update.sh /opt/ ADD namespace.yaml /opt/ -ADD kubectl /usr/local/bin/kubectl +ADD kubectl /usr/local/bin/ -CMD /opt/kube-addons.sh +CMD ["/opt/kube-addons.sh"] diff --git a/cluster/addons/addon-manager/Makefile b/cluster/addons/addon-manager/Makefile index bdd5f2a8ec0..5898b90eb38 100644 --- a/cluster/addons/addon-manager/Makefile +++ b/cluster/addons/addon-manager/Makefile @@ -13,22 +13,62 @@ # limitations under the License. IMAGE=gcr.io/google-containers/kube-addon-manager -VERSION=v1 -KUBECTL_VERSION=v1.2.3 +ARCH?=amd64 +TEMP_DIR:=$(shell mktemp -d) +VERSION=v2 -.PHONY: build push container +# amd64 and arm has "stable" binaries pushed for v1.2, arm64 and ppc64le hasn't so they have to fetch the latest alpha +# however, arm64 and ppc64le are very experimental right now, so it's okay +ifeq ($(ARCH),amd64) + KUBECTL_VERSION?=v1.2.4 + BASEIMAGE?=python:2.7-slim +endif +ifeq ($(ARCH),arm) + KUBECTL_VERSION?=v1.2.4 + BASEIMAGE?=hypriot/rpi-python:2.7 + QEMUARCH=arm +endif +ifeq ($(ARCH),arm64) + KUBECTL_VERSION?=v1.3.0-alpha.3 + BASEIMAGE?=aarch64/python:2.7-slim + QEMUARCH=aarch64 +endif +ifeq ($(ARCH),ppc64le) + KUBECTL_VERSION?=v1.3.0-alpha.3 + BASEIMAGE?=ppc64le/python:2.7-slim + QEMUARCH=ppc64le +endif -build: kubectl - docker build -t "$(IMAGE):$(VERSION)" . +.PHONY: build push -kubectl: - curl "https://storage.googleapis.com/kubernetes-release/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl" \ - -o kubectl - chmod +x kubectl +all: build +build: + cp ./* $(TEMP_DIR) + curl -sSL --retry 5 https://storage.googleapis.com/kubernetes-release/release/$(KUBECTL_VERSION)/bin/linux/$(ARCH)/kubectl > $(TEMP_DIR)/kubectl + chmod +x $(TEMP_DIR)/kubectl + cd ${TEMP_DIR} && sed -i.back "s|ARCH|$(QEMUARCH)|g" Dockerfile + cd $(TEMP_DIR) && sed -i.back "s|BASEIMAGE|$(BASEIMAGE)|g" Dockerfile + +ifeq ($(ARCH),amd64) + # When building "normally" for amd64, remove the whole line, it has no part in the amd64 image + cd $(TEMP_DIR) && sed -i "/CROSS_BUILD_/d" Dockerfile +else + # When cross-building, only the placeholder "CROSS_BUILD_" should be removed + # Register /usr/bin/qemu-ARCH-static as the handler for other-arch binaries in the kernel + docker run --rm --privileged multiarch/qemu-user-static:register --reset + curl -sSL --retry 5 https://github.com/multiarch/qemu-user-static/releases/download/v2.5.0/x86_64_qemu-$(QEMUARCH)-static.tar.xz | tar -xJ -C $(TEMP_DIR) + cd $(TEMP_DIR) && sed -i "s/CROSS_BUILD_//g" Dockerfile +endif + + docker build -t $(IMAGE)-$(ARCH):$(VERSION) $(TEMP_DIR) push: build - gcloud docker push "$(IMAGE):$(VERSION)" + gcloud docker push $(IMAGE)-$(ARCH):$(VERSION) +ifeq ($(ARCH),amd64) + # Backward compatibility. TODO: deprecate this image tag + docker tag -f $(IMAGE)-$(ARCH):$(VERSION) $(IMAGE):$(VERSION) + gcloud docker push $(IMAGE):$(VERSION) +endif clean: - rm kubectl - docker rmi -f "$(IMAGE):$(VERSION)" + docker rmi -f $(IMAGE)-$(ARCH):$(VERSION) diff --git a/cluster/addons/addon-manager/README.md b/cluster/addons/addon-manager/README.md new file mode 100644 index 00000000000..a9458aa030c --- /dev/null +++ b/cluster/addons/addon-manager/README.md @@ -0,0 +1,37 @@ +### addon-manager + +The `addon-manager` periodically checks for Kubernetes manifest changes in the `/etc/kubernetes/addons` directory, +and when there's a new or changed addon, the `addon-manager` automatically `kubectl create`s it. + +It supports `ReplicationControllers`, `Deployments`, `DaemonSets`, `Services`, `PersistentVolumes` and `PersistentVolumeClaims`. + +The `addon-manager` is built for multiple architectures. + +#### How to release + +1. Change something in the source +2. Bump `VERSION` in the `Makefile` +3. Bump `KUBECTL_VERSION` in the `Makefile` if required +4. Build the `amd64` image and test it on a cluster +5. Push all images + +```console +# Build for linux/amd64 (default) +$ make push ARCH=amd64 +# ---> gcr.io/google-containers/kube-addon-manager-amd64:VERSION +# ---> gcr.io/google-containers/kube-addon-manager:VERSION (image with backwards-compatible naming) + +$ make push ARCH=arm +# ---> gcr.io/google-containers/kube-addon-manager-arm:VERSION + +$ make push ARCH=arm64 +# ---> gcr.io/google-containers/kube-addon-manager-arm64:VERSION + +$ make push ARCH=ppc64le +# ---> gcr.io/google-containers/kube-addon-manager-ppc64le:VERSION +``` + +If you don't want to push the images, run `make` or `make build` instead + + +[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/cluster/addons/addon-manager/README.md?pixel)]() diff --git a/cluster/addons/addon-manager/kube-addon-update.sh b/cluster/addons/addon-manager/kube-addon-update.sh index dcaf3518c69..8b38b5ea3a3 100755 --- a/cluster/addons/addon-manager/kube-addon-update.sh +++ b/cluster/addons/addon-manager/kube-addon-update.sh @@ -198,7 +198,7 @@ function run-until-success() { # returns a list of / pairs (nsnames) function get-addon-nsnames-from-server() { local -r obj_type=$1 - "${KUBECTL}" get "${obj_type}" --all-namespaces -o go-template="{{range.items}}{{.metadata.namespace}}/{{.metadata.name}} {{end}}" --api-version=v1 -l kubernetes.io/cluster-service=true + "${KUBECTL}" get "${obj_type}" --all-namespaces -o go-template="{{range.items}}{{.metadata.namespace}}/{{.metadata.name}} {{end}}" -l kubernetes.io/cluster-service=true } # returns the characters after the last separator (including) @@ -476,6 +476,7 @@ function update-addons() { # be careful, reconcile-objects uses global variables reconcile-objects ${addon_path} ReplicationController "-" & reconcile-objects ${addon_path} Deployment "-" & + reconcile-objects ${addon_path} DaemonSet "-" & # We don't expect names to be versioned for the following kinds, so # we match the entire name, ignoring version suffix. From 73947cc5aa6a107bde9e4ce463ad9bbe69d78253 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lucas=20K=C3=A4ldstr=C3=B6m?= Date: Fri, 20 May 2016 19:28:13 +0300 Subject: [PATCH 2/2] Large changes to the docker deployment. Added kube-addon-manager as a static pod. The addon-manager deploys kube-proxy as a DaemonSet as well as Dashboard and DNS automatically. SecurityContextDeny is removed from the manifests. Also, the turnup.sh and turndown.sh scripts are removed because we don't need them anymore, they're covered by the online documentation --- .../dashboard/dashboard-controller.yaml | 4 +- .../addons/dashboard/dashboard-service.yaml | 2 + cluster/addons/dns/skydns-rc.yaml.in | 1 + cluster/addons/dns/skydns-svc.yaml.in | 1 + cluster/images/hyperkube/Dockerfile | 19 ++- cluster/images/hyperkube/Makefile | 23 ++- .../images/hyperkube/addons/dashboard-rc.yaml | 51 +++++++ .../dashboard-svc.yaml} | 34 ++--- cluster/images/hyperkube/addons/dns-rc.yaml | 144 ++++++++++++++++++ cluster/images/hyperkube/addons/dns-svc.yaml | 35 +++++ .../images/hyperkube/addons/kube-proxy.yaml | 43 ++++++ cluster/images/hyperkube/copy-addons.sh | 31 ++++ cluster/images/hyperkube/kube-proxy.json | 27 ---- cluster/images/hyperkube/setup-files.sh | 1 - .../hyperkube/static-pods/addon-manager.json | 51 +++++++ .../hyperkube/{ => static-pods}/etcd.json | 0 .../{ => static-pods}/master-multi.json | 4 +- .../hyperkube/{ => static-pods}/master.json | 4 +- cluster/images/hyperkube/turnup.sh | 49 ------ 19 files changed, 407 insertions(+), 117 deletions(-) create mode 100644 cluster/images/hyperkube/addons/dashboard-rc.yaml rename cluster/images/hyperkube/{teardown.sh => addons/dashboard-svc.yaml} (56%) mode change 100755 => 100644 create mode 100644 cluster/images/hyperkube/addons/dns-rc.yaml create mode 100644 cluster/images/hyperkube/addons/dns-svc.yaml create mode 100644 cluster/images/hyperkube/addons/kube-proxy.yaml create mode 100755 cluster/images/hyperkube/copy-addons.sh delete mode 100644 cluster/images/hyperkube/kube-proxy.json create mode 100644 cluster/images/hyperkube/static-pods/addon-manager.json rename cluster/images/hyperkube/{ => static-pods}/etcd.json (100%) rename cluster/images/hyperkube/{ => static-pods}/master-multi.json (96%) rename cluster/images/hyperkube/{ => static-pods}/master.json (96%) delete mode 100755 cluster/images/hyperkube/turnup.sh diff --git a/cluster/addons/dashboard/dashboard-controller.yaml b/cluster/addons/dashboard/dashboard-controller.yaml index 3b46c319df2..7928203c9c1 100644 --- a/cluster/addons/dashboard/dashboard-controller.yaml +++ b/cluster/addons/dashboard/dashboard-controller.yaml @@ -1,8 +1,8 @@ +# This file should be kept in sync with cluster/images/hyperkube/dashboard-rc.yaml +# and cluster/gce/coreos/kube-manifests/addons/dashboard/dashboard-controller.yaml apiVersion: v1 kind: ReplicationController metadata: - # Keep the name in sync with image version and - # gce/coreos/kube-manifests/addons/dashboard counterparts name: kubernetes-dashboard-v1.0.1 namespace: kube-system labels: diff --git a/cluster/addons/dashboard/dashboard-service.yaml b/cluster/addons/dashboard/dashboard-service.yaml index 195b503de10..d9aabeccce5 100644 --- a/cluster/addons/dashboard/dashboard-service.yaml +++ b/cluster/addons/dashboard/dashboard-service.yaml @@ -1,3 +1,5 @@ +# This file should be kept in sync with cluster/images/hyperkube/dashboard-svc.yaml +# and cluster/gce/coreos/kube-manifests/addons/dashboard/dashboard-service.yaml apiVersion: v1 kind: Service metadata: diff --git a/cluster/addons/dns/skydns-rc.yaml.in b/cluster/addons/dns/skydns-rc.yaml.in index 0b29293bd30..6498b89edfa 100644 --- a/cluster/addons/dns/skydns-rc.yaml.in +++ b/cluster/addons/dns/skydns-rc.yaml.in @@ -1,3 +1,4 @@ +# This file should be kept in sync with cluster/images/hyperkube/dns-rc.yaml apiVersion: v1 kind: ReplicationController metadata: diff --git a/cluster/addons/dns/skydns-svc.yaml.in b/cluster/addons/dns/skydns-svc.yaml.in index 242c8871eec..323605c0c45 100644 --- a/cluster/addons/dns/skydns-svc.yaml.in +++ b/cluster/addons/dns/skydns-svc.yaml.in @@ -1,3 +1,4 @@ +# This file should be kept in sync with cluster/images/hyperkube/dns-svc.yaml apiVersion: v1 kind: Service metadata: diff --git a/cluster/images/hyperkube/Dockerfile b/cluster/images/hyperkube/Dockerfile index 7a247f2b93d..6847d635efd 100644 --- a/cluster/images/hyperkube/Dockerfile +++ b/cluster/images/hyperkube/Dockerfile @@ -38,25 +38,28 @@ RUN cp /usr/bin/nsenter /nsenter COPY hyperkube /hyperkube # Manifests for the docker guide -COPY master.json /etc/kubernetes/manifests/master.json -COPY etcd.json /etc/kubernetes/manifests/etcd.json -COPY kube-proxy.json /etc/kubernetes/manifests/kube-proxy.json +COPY static-pods/master.json /etc/kubernetes/manifests/ +COPY static-pods/etcd.json /etc/kubernetes/manifests/ +COPY static-pods/addon-manager.json /etc/kubernetes/manifests/ # Manifests for the docker-multinode guide -COPY master-multi.json /etc/kubernetes/manifests-multi/master.json -COPY kube-proxy.json /etc/kubernetes/manifests-multi/kube-proxy.json +COPY static-pods/master-multi.json /etc/kubernetes/manifests-multi/ +COPY static-pods/addon-manager.json /etc/kubernetes/manifests-multi/ + +# Copy over all addons +COPY addons /etc/kubernetes/addons # Other required scripts for the setup COPY safe_format_and_mount /usr/share/google/safe_format_and_mount COPY setup-files.sh /setup-files.sh COPY make-ca-cert.sh /make-ca-cert.sh +COPY copy-addons.sh /copy-addons.sh # easy-rsa package required by make-ca-cert ADD https://storage.googleapis.com/kubernetes-release/easy-rsa/easy-rsa.tar.gz /root/kube/ -RUN mkdir -p /opt/cni -RUN curl https://storage.googleapis.com/kubernetes-release/network-plugins/cni-c864f0e1ea73719b8f4582402b0847064f9883b0.tar.gz \ - | tar xzv -C /opt/cni +# Copy the cni folder into /opt/ +COPY cni /opt/cni # Create symlinks for each hyperkube server # TODO: this is unreliable for now (e.g. running "/kubelet" panics) diff --git a/cluster/images/hyperkube/Makefile b/cluster/images/hyperkube/Makefile index 5d600324ff1..15576974a04 100644 --- a/cluster/images/hyperkube/Makefile +++ b/cluster/images/hyperkube/Makefile @@ -20,6 +20,7 @@ REGISTRY?="gcr.io/google_containers" ARCH?=amd64 TEMP_DIR:=$(shell mktemp -d) +CNI_RELEASE=c864f0e1ea73719b8f4582402b0847064f9883b0 UNAME_S:=$(shell uname -s) ifeq ($(UNAME_S),Darwin) @@ -28,6 +29,7 @@ endif ifeq ($(UNAME_S),Linux) SED_CMD?=sed -i endif + ifeq ($(ARCH),amd64) BASEIMAGE?=debian:jessie endif @@ -51,27 +53,32 @@ build: ifndef VERSION $(error VERSION is undefined) endif - cp ./* ${TEMP_DIR} + cp -r ./* ${TEMP_DIR} + mkdir -p ${TEMP_DIR}/cni cp ../../saltbase/salt/helpers/safe_format_and_mount ${TEMP_DIR} cp ../../saltbase/salt/generate-cert/make-ca-cert.sh ${TEMP_DIR} cp ../../../_output/dockerized/bin/linux/${ARCH}/hyperkube ${TEMP_DIR} - cd ${TEMP_DIR} && sed -i.back "s|VERSION|${VERSION}|g" master-multi.json master.json kube-proxy.json - cd ${TEMP_DIR} && sed -i.back "s|ARCH|${ARCH}|g" master-multi.json master.json kube-proxy.json etcd.json + + cd ${TEMP_DIR} && sed -i.back "s|VERSION|${VERSION}|g" addons/*.yaml static-pods/*.json + cd ${TEMP_DIR} && sed -i.back "s|ARCH|${ARCH}|g" addons/*.yaml static-pods/*.json cd ${TEMP_DIR} && sed -i.back "s|ARCH|${QEMUARCH}|g" Dockerfile cd ${TEMP_DIR} && sed -i.back "s|BASEIMAGE|${BASEIMAGE}|g" Dockerfile - rm ${TEMP_DIR}/*.back + rm ${TEMP_DIR}/addons/*.back # Make scripts executable before they are copied into the Docker image. If we make them executable later, in another layer # they'll take up twice the space because the new executable binary differs from the old one, but everything is cached in layers. cd ${TEMP_DIR} && chmod a+rx \ - hyperkube \ - safe_format_and_mount \ - setup-files.sh \ - make-ca-cert.sh + hyperkube \ + safe_format_and_mount \ + setup-files.sh \ + make-ca-cert.sh \ + copy-addons.sh ifeq ($(ARCH),amd64) # When building "normally" for amd64, remove the whole line, it has no part in the amd64 image cd ${TEMP_DIR} && ${SED_CMD} "/CROSS_BUILD_/d" Dockerfile + # Download CNI + curl -sSL --retry 5 https://storage.googleapis.com/kubernetes-release/network-plugins/cni-${CNI_RELEASE}.tar.gz | tar -xz -C ${TEMP_DIR}/cni else # When cross-building, only the placeholder "CROSS_BUILD_" should be removed # Register /usr/bin/qemu-ARCH-static as the handler for ARM binaries in the kernel diff --git a/cluster/images/hyperkube/addons/dashboard-rc.yaml b/cluster/images/hyperkube/addons/dashboard-rc.yaml new file mode 100644 index 00000000000..cda866e6c0c --- /dev/null +++ b/cluster/images/hyperkube/addons/dashboard-rc.yaml @@ -0,0 +1,51 @@ +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file should be kept in sync with cluster/addons/dashboard/dashboard-controller.yaml +apiVersion: v1 +kind: ReplicationController +metadata: + name: kubernetes-dashboard + namespace: kube-system + labels: + app: kubernetes-dashboard + version: v1.0.1 + kubernetes.io/cluster-service: "true" +spec: + replicas: 1 + selector: + app: kubernetes-dashboard + version: v1.0.1 + kubernetes.io/cluster-service: "true" + template: + metadata: + labels: + app: kubernetes-dashboard + version: v1.0.1 + kubernetes.io/cluster-service: "true" + spec: + containers: + - name: kubernetes-dashboard + # ARCH will be replaced with the architecture it's built for. Check out the Makefile for more details + image: gcr.io/google_containers/kubernetes-dashboard-ARCH:v1.0.1 + imagePullPolicy: Always + ports: + - containerPort: 9090 + protocol: TCP + livenessProbe: + httpGet: + path: / + port: 9090 + initialDelaySeconds: 30 + timeoutSeconds: 30 diff --git a/cluster/images/hyperkube/teardown.sh b/cluster/images/hyperkube/addons/dashboard-svc.yaml old mode 100755 new mode 100644 similarity index 56% rename from cluster/images/hyperkube/teardown.sh rename to cluster/images/hyperkube/addons/dashboard-svc.yaml index f94a8a1041a..e856b35ca98 --- a/cluster/images/hyperkube/teardown.sh +++ b/cluster/images/hyperkube/addons/dashboard-svc.yaml @@ -1,6 +1,4 @@ -#!/bin/bash - -# Copyright 2015 The Kubernetes Authors All rights reserved. +# Copyright 2016 The Kubernetes Authors All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,18 +12,18 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Tears down an existing cluster. Warning destroys _all_ docker containers on the machine - -set -o errexit -set -o nounset -set -o pipefail - -echo "Warning, this will delete all Docker containers on this machine." -echo "Proceed? [Y/n]" - -read resp -if [[ $resp == "n" || $resp == "N" ]]; then - exit 0 -fi - -docker ps -aq | xargs docker rm -f +# This file should be kept in sync with cluster/addons/dashboard/dashboard-service.yaml +kind: Service +apiVersion: v1 +metadata: + name: kubernetes-dashboard + namespace: kube-system + labels: + app: kubernetes-dashboard + kubernetes.io/cluster-service: "true" +spec: + ports: + - port: 80 + targetPort: 9090 + selector: + app: kubernetes-dashboard diff --git a/cluster/images/hyperkube/addons/dns-rc.yaml b/cluster/images/hyperkube/addons/dns-rc.yaml new file mode 100644 index 00000000000..2d609b3096a --- /dev/null +++ b/cluster/images/hyperkube/addons/dns-rc.yaml @@ -0,0 +1,144 @@ +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file should be kept in sync with cluster/addons/dns/skydns-rc.yaml.in +apiVersion: v1 +kind: ReplicationController +metadata: + name: kube-dns-v11 + namespace: kube-system + labels: + k8s-app: kube-dns + version: v11 + kubernetes.io/cluster-service: "true" +spec: + replicas: 1 + selector: + k8s-app: kube-dns + version: v11 + template: + metadata: + labels: + k8s-app: kube-dns + version: v11 + kubernetes.io/cluster-service: "true" + spec: + containers: + - name: etcd + # ARCH will be replaced with the architecture it's built for. Check out the Makefile for more details + image: gcr.io/google_containers/etcd-ARCH:2.2.5 + resources: + # TODO: Set memory limits when we've profiled the container for large + # clusters, then set request = limit to keep this container in + # guaranteed class. Currently, this container falls into the + # "burstable" category so the kubelet doesn't backoff from restarting it. + limits: + cpu: 100m + memory: 500Mi + requests: + cpu: 100m + memory: 50Mi + command: + - /usr/local/bin/etcd + - -data-dir + - /var/etcd/data + - -listen-client-urls + - http://127.0.0.1:2379,http://127.0.0.1:4001 + - -advertise-client-urls + - http://127.0.0.1:2379,http://127.0.0.1:4001 + - -initial-cluster-token + - skydns-etcd + volumeMounts: + - name: etcd-storage + mountPath: /var/etcd/data + - name: kube2sky + image: gcr.io/google_containers/kube2sky-ARCH:1.15 + resources: + # TODO: Set memory limits when we've profiled the container for large + # clusters, then set request = limit to keep this container in + # guaranteed class. Currently, this container falls into the + # "burstable" category so the kubelet doesn't backoff from restarting it. + limits: + cpu: 100m + # Kube2sky watches all pods. + memory: 200Mi + requests: + cpu: 100m + memory: 50Mi + livenessProbe: + httpGet: + path: /healthz + port: 8080 + scheme: HTTP + initialDelaySeconds: 60 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 5 + readinessProbe: + httpGet: + path: /readiness + port: 8081 + scheme: HTTP + # we poll on pod startup for the Kubernetes master service and + # only setup the /readiness HTTP server once that's available. + initialDelaySeconds: 30 + timeoutSeconds: 5 + args: + # command = "/kube2sky" + - --domain=cluster.local + - name: skydns + image: gcr.io/google_containers/skydns-ARCH:1.0 + resources: + # TODO: Set memory limits when we've profiled the container for large + # clusters, then set request = limit to keep this container in + # guaranteed class. Currently, this container falls into the + # "burstable" category so the kubelet doesn't backoff from restarting it. + limits: + cpu: 100m + memory: 200Mi + requests: + cpu: 100m + memory: 50Mi + args: + - -machines=http://127.0.0.1:4001 + - -addr=0.0.0.0:53 + - -ns-rotate=false + - -domain=cluster.local. + ports: + - containerPort: 53 + name: dns + protocol: UDP + - containerPort: 53 + name: dns-tcp + protocol: TCP + - name: healthz + image: gcr.io/google_containers/exechealthz-ARCH:1.0 + resources: + # keep request = limit to keep this container in guaranteed class + limits: + cpu: 10m + memory: 20Mi + requests: + cpu: 10m + memory: 20Mi + args: + - -cmd=nslookup kubernetes.default.svc.cluster.local 127.0.0.1 >/dev/null + - -port=8080 + ports: + - containerPort: 8080 + protocol: TCP + volumes: + - name: etcd-storage + emptyDir: {} + dnsPolicy: Default # Don't use cluster DNS. diff --git a/cluster/images/hyperkube/addons/dns-svc.yaml b/cluster/images/hyperkube/addons/dns-svc.yaml new file mode 100644 index 00000000000..4c08c2d2ec4 --- /dev/null +++ b/cluster/images/hyperkube/addons/dns-svc.yaml @@ -0,0 +1,35 @@ +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file should be kept in sync with cluster/addons/dns/skydns-svc.yaml.in +apiVersion: v1 +kind: Service +metadata: + name: kube-dns + namespace: kube-system + labels: + k8s-app: kube-dns + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "KubeDNS" +spec: + selector: + k8s-app: kube-dns + clusterIP: 10.0.0.10 + ports: + - name: dns + port: 53 + protocol: UDP + - name: dns-tcp + port: 53 + protocol: TCP diff --git a/cluster/images/hyperkube/addons/kube-proxy.yaml b/cluster/images/hyperkube/addons/kube-proxy.yaml new file mode 100644 index 00000000000..03790196c58 --- /dev/null +++ b/cluster/images/hyperkube/addons/kube-proxy.yaml @@ -0,0 +1,43 @@ +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: k8s-proxy + namespace: kube-system + labels: + k8s-app: k8s-proxy + version: v1 + kubernetes.io/cluster-service: "true" +spec: + template: + metadata: + labels: + k8s-app: k8s-proxy + version: v1 + kubernetes.io/cluster-service: "true" + spec: + hostNetwork: true + containers: + - name: kube-proxy + image: gcr.io/google_containers/hyperkube-ARCH:VERSION + command: + - /hyperkube + - proxy + - --master=http://127.0.0.1:8080 + - --v=2 + - --resource-container="" + securityContext: + privileged: true diff --git a/cluster/images/hyperkube/copy-addons.sh b/cluster/images/hyperkube/copy-addons.sh new file mode 100755 index 00000000000..499f04d7683 --- /dev/null +++ b/cluster/images/hyperkube/copy-addons.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# Now we're running in the sidecar container +# /etc/kubernetes/addons holds the data in the hyperkube container +# /srv/kubernetes is an emptyDir that maps to /etc/kubernetes in the addon-manager container +# This way we're using the latest manifests from hyperkube without updating +# kube-addon-manager which is used for other deployments too + +# While there is no data copied over to the emptyDir, try to copy it. +while [[ ! -d /srv/kubernetes/addons ]]; do + cp -r /etc/kubernetes/* /srv/kubernetes/ +done + +# Then sleep forever +while true; do + sleep 3600; +done diff --git a/cluster/images/hyperkube/kube-proxy.json b/cluster/images/hyperkube/kube-proxy.json deleted file mode 100644 index b005433b86d..00000000000 --- a/cluster/images/hyperkube/kube-proxy.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "name": "k8s-proxy", - "namespace": "kube-system" - }, - "spec": { - "hostNetwork": true, - "containers": [ - { - "name": "kube-proxy", - "image": "gcr.io/google_containers/hyperkube-ARCH:VERSION", - "command": [ - "/hyperkube", - "proxy", - "--master=http://127.0.0.1:8080", - "--v=2", - "--resource-container=\"\"" - ], - "securityContext": { - "privileged": true - } - } - ] - } -} diff --git a/cluster/images/hyperkube/setup-files.sh b/cluster/images/hyperkube/setup-files.sh index c9d80484377..395ab7bb7f0 100644 --- a/cluster/images/hyperkube/setup-files.sh +++ b/cluster/images/hyperkube/setup-files.sh @@ -1,5 +1,4 @@ #!/bin/bash - # Copyright 2015 The Kubernetes Authors All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/cluster/images/hyperkube/static-pods/addon-manager.json b/cluster/images/hyperkube/static-pods/addon-manager.json new file mode 100644 index 00000000000..3a79a27d998 --- /dev/null +++ b/cluster/images/hyperkube/static-pods/addon-manager.json @@ -0,0 +1,51 @@ +{ + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "name": "kube-addon-manager", + "namespace": "kube-system", + "version": "v1" + }, + "spec": { + "hostNetwork": true, + "containers": [ + { + "name": "kube-addon-manager", + "image": "gcr.io/google-containers/kube-addon-manager-ARCH:v2", + "resources": { + "requests": { + "cpu": "5m", + "memory": "50Mi" + } + }, + "volumeMounts": [ + { + "name": "addons", + "mountPath": "/etc/kubernetes/", + "readOnly": true + } + ] + }, + { + "name": "kube-addon-manager-data", + "image": "gcr.io/google_containers/hyperkube-ARCH:VERSION", + "command": [ + "/copy-addons.sh" + ], + "volumeMounts": [ + { + "name": "addons", + "mountPath": "/srv/kubernetes/", + "readOnly": false + } + ] + } + ], + "volumes":[ + { + "name": "addons", + "emptyDir": {} + } + ] + } +} diff --git a/cluster/images/hyperkube/etcd.json b/cluster/images/hyperkube/static-pods/etcd.json similarity index 100% rename from cluster/images/hyperkube/etcd.json rename to cluster/images/hyperkube/static-pods/etcd.json diff --git a/cluster/images/hyperkube/master-multi.json b/cluster/images/hyperkube/static-pods/master-multi.json similarity index 96% rename from cluster/images/hyperkube/master-multi.json rename to cluster/images/hyperkube/static-pods/master-multi.json index 3e201fdd486..684b186d361 100644 --- a/cluster/images/hyperkube/master-multi.json +++ b/cluster/images/hyperkube/static-pods/master-multi.json @@ -36,7 +36,7 @@ "--service-cluster-ip-range=10.0.0.1/24", "--insecure-bind-address=0.0.0.0", "--etcd-servers=http://127.0.0.1:4001", - "--admission-control=NamespaceLifecycle,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota", + "--admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,ResourceQuota", "--client-ca-file=/srv/kubernetes/ca.crt", "--basic-auth-file=/srv/kubernetes/basic_auth.csv", "--min-request-timeout=300", @@ -44,7 +44,7 @@ "--tls-private-key-file=/srv/kubernetes/server.key", "--token-auth-file=/srv/kubernetes/known_tokens.csv", "--allow-privileged=true", - "--v=4" + "--v=2" ], "volumeMounts": [ { diff --git a/cluster/images/hyperkube/master.json b/cluster/images/hyperkube/static-pods/master.json similarity index 96% rename from cluster/images/hyperkube/master.json rename to cluster/images/hyperkube/static-pods/master.json index ca8cd8f4d24..5b15757bf4d 100644 --- a/cluster/images/hyperkube/master.json +++ b/cluster/images/hyperkube/static-pods/master.json @@ -36,7 +36,7 @@ "--service-cluster-ip-range=10.0.0.1/24", "--insecure-bind-address=127.0.0.1", "--etcd-servers=http://127.0.0.1:4001", - "--admission-control=NamespaceLifecycle,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota", + "--admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,ResourceQuota", "--client-ca-file=/srv/kubernetes/ca.crt", "--basic-auth-file=/srv/kubernetes/basic_auth.csv", "--min-request-timeout=300", @@ -44,7 +44,7 @@ "--tls-private-key-file=/srv/kubernetes/server.key", "--token-auth-file=/srv/kubernetes/known_tokens.csv", "--allow-privileged=true", - "--v=4" + "--v=2" ], "volumeMounts": [ { diff --git a/cluster/images/hyperkube/turnup.sh b/cluster/images/hyperkube/turnup.sh deleted file mode 100755 index b19f9e9a954..00000000000 --- a/cluster/images/hyperkube/turnup.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/bash - -# Copyright 2015 The Kubernetes Authors All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Useful for testing images and changes, turns up a fresh single node cluster - -set -o errexit -set -o nounset -set -o pipefail - -K8S_VERSION=${K8S_VERSION:-"1.2.0"} - -docker run \ - --volume=/:/rootfs:ro \ - --volume=/sys:/sys:ro \ - --volume=/var/lib/docker/:/var/lib/docker:rw \ - --volume=/var/lib/kubelet/:/var/lib/kubelet:rw \ - --volume=/var/run:/var/run:rw \ - --net=host \ - --pid=host \ - --privileged=true \ - -d gcr.io/google_containers/hyperkube-amd64:v${K8S_VERSION} \ - /hyperkube kubelet \ - --containerized \ - --hostname-override="127.0.0.1" \ - --address="0.0.0.0" \ - --api-servers=http://localhost:8080 \ - --config=/etc/kubernetes/manifests \ - --cluster-dns=10.0.0.10 \ - --cluster-domain=cluster.local \ - --allow-privileged=true --v=2 - -until $(kubectl cluster-info &> /dev/null); do - sleep 1 -done - -kubectl create ns kube-system