From d2284fee88613c2b108512f95a804b99fd1013a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9na=C3=AFc=20Huard?= Date: Thu, 23 Apr 2015 17:23:23 +0200 Subject: [PATCH 1/3] Revert "libvirt-coreos cluster: Fix etcd versions incompatibility issue" This reverts commit 11556dc927247b96d1a18e45e48876e75ce3edab. --- cluster/libvirt-coreos/.gitignore | 1 - cluster/libvirt-coreos/coreos.xml | 5 ----- cluster/libvirt-coreos/user_data.yml | 24 +----------------------- cluster/libvirt-coreos/util.sh | 18 ------------------ 4 files changed, 1 insertion(+), 47 deletions(-) diff --git a/cluster/libvirt-coreos/.gitignore b/cluster/libvirt-coreos/.gitignore index 3eb6823b0f2..f26d8e4020a 100644 --- a/cluster/libvirt-coreos/.gitignore +++ b/cluster/libvirt-coreos/.gitignore @@ -1,3 +1,2 @@ /libvirt_storage_pool/ /coreos_production_qemu_image.img.bz2 -/etcd-v2.0.9-linux-amd64.tar.gz diff --git a/cluster/libvirt-coreos/coreos.xml b/cluster/libvirt-coreos/coreos.xml index a0490e2eadd..041f3a01510 100644 --- a/cluster/libvirt-coreos/coreos.xml +++ b/cluster/libvirt-coreos/coreos.xml @@ -35,11 +35,6 @@ - - - - - diff --git a/cluster/libvirt-coreos/user_data.yml b/cluster/libvirt-coreos/user_data.yml index 9df5eeea466..4cbd93216eb 100644 --- a/cluster/libvirt-coreos/user_data.yml +++ b/cluster/libvirt-coreos/user_data.yml @@ -17,22 +17,11 @@ coreos: etcd: name: ${name} addr: ${public_ip}:4001 - # bind-addr: 0.0.0.0 + bind-addr: 0.0.0.0 peer-addr: ${public_ip}:7001 # peers: {etcd_peers} discovery: ${discovery} units: - - name: etcd.service - drop-ins: - - name: opt-etcd2.conf - content: | - [Unit] - After=opt-etcd.mount - Requires=opt-etcd.mount - - [Service] - ExecStart= - ExecStart=/opt/etcd/bin/etcd - name: static.network command: start content: | @@ -114,17 +103,6 @@ coreos: Where=/opt/kubernetes Options=ro,trans=virtio,version=9p2000.L Type=9p - - name: opt-etcd.mount - command: start - content: | - [Unit] - ConditionVirtualization=|vm - - [Mount] - What=etcd - Where=/opt/etcd - Options=ro,trans=virtio,version=9p2000.L - Type=9p update: group: ${COREOS_CHANNEL:-alpha} reboot-strategy: off diff --git a/cluster/libvirt-coreos/util.sh b/cluster/libvirt-coreos/util.sh index 828adf995ee..fc3547e7f17 100644 --- a/cluster/libvirt-coreos/util.sh +++ b/cluster/libvirt-coreos/util.sh @@ -26,8 +26,6 @@ export LIBVIRT_DEFAULT_URI=qemu:///system readonly POOL=kubernetes readonly POOL_PATH="$(cd $ROOT && pwd)/libvirt_storage_pool" -ETCD_VERSION=${ETCD_VERSION:-v2.0.9} - # join # Concatenates the list elements with the delimiter passed as first parameter # @@ -96,9 +94,6 @@ function destroy-pool { virsh vol-delete $vol --pool $POOL done - rm -rf "$POOL_PATH"/etcd/* - virsh vol-delete etcd --pool $POOL || true - [[ "$1" == 'keep_base_image' ]] && return set +e @@ -146,18 +141,6 @@ function initialize-pool { render-template "$ROOT/skydns-rc.yaml" > "$POOL_PATH/kubernetes/addons/skydns-rc.yaml" fi - mkdir -p "$POOL_PATH/etcd" - if [[ ! -f "$ROOT/etcd-${ETCD_VERSION}-linux-amd64.tar.gz" ]]; then - wget -P "$ROOT" https://github.com/coreos/etcd/releases/download/${ETCD_VERSION}/etcd-${ETCD_VERSION}-linux-amd64.tar.gz - fi - if [[ "$ROOT/etcd-${ETCD_VERSION}-linux-amd64.tar.gz" -nt "$POOL_PATH/etcd/etcd" ]]; then - tar -x -C "$POOL_PATH/etcd" -f "$ROOT/etcd-${ETCD_VERSION}-linux-amd64.tar.gz" etcd-${ETCD_VERSION}-linux-amd64 - rm -rf "$POOL_PATH/etcd/bin/*" - mkdir -p "$POOL_PATH/etcd/bin" - mv "$POOL_PATH"/etcd/etcd-${ETCD_VERSION}-linux-amd64/{etcd,etcdctl} "$POOL_PATH/etcd/bin" - rm -rf "$POOL_PATH/etcd/etcd-${ETCD_VERSION}-linux-amd64" - fi - virsh pool-refresh $POOL } @@ -205,7 +188,6 @@ function kube-up { readonly ssh_keys="$(cat ~/.ssh/id_*.pub | sed 's/^/ - /')" readonly kubernetes_dir="$POOL_PATH/kubernetes" - readonly etcd_dir="$POOL_PATH/etcd" readonly discovery=$(curl -s https://discovery.etcd.io/new) readonly machines=$(join , "${KUBE_MINION_IP_ADDRESSES[@]}") From 35452538f4f9337c2746bfcc4e39be51f8762264 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9na=C3=AFc=20Huard?= Date: Thu, 23 Apr 2015 18:22:00 +0200 Subject: [PATCH 2/3] Switch to etcd 2 --- cluster/libvirt-coreos/user_data.yml | 13 ++++++------- cluster/libvirt-coreos/user_data_master.yml | 6 +++--- cluster/libvirt-coreos/user_data_minion.yml | 8 ++++---- cluster/libvirt-coreos/util.sh | 3 ++- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/cluster/libvirt-coreos/user_data.yml b/cluster/libvirt-coreos/user_data.yml index 4cbd93216eb..4616c24d44c 100644 --- a/cluster/libvirt-coreos/user_data.yml +++ b/cluster/libvirt-coreos/user_data.yml @@ -14,13 +14,12 @@ write_files: RuntimeMaxUse=50M coreos: - etcd: - name: ${name} - addr: ${public_ip}:4001 - bind-addr: 0.0.0.0 - peer-addr: ${public_ip}:7001 - # peers: {etcd_peers} + etcd2: discovery: ${discovery} + advertise-client-urls: http://${public_ip}:2379 + initial-advertise-peer-urls: http://${public_ip}:2380 + listen-client-urls: http://0.0.0.0:2379 + listen-peer-urls: http://${public_ip}:2380 units: - name: static.network command: start @@ -69,7 +68,7 @@ coreos: ExecStart=/usr/sbin/iptables -w -t nat -A POSTROUTING -o eth0 -j MASQUERADE ! -d ${CONTAINER_SUBNET} RemainAfterExit=yes Type=oneshot - - name: etcd.service + - name: etcd2.service command: start - name: docker.service command: start diff --git a/cluster/libvirt-coreos/user_data_master.yml b/cluster/libvirt-coreos/user_data_master.yml index cd48387b278..c34b034fe33 100644 --- a/cluster/libvirt-coreos/user_data_master.yml +++ b/cluster/libvirt-coreos/user_data_master.yml @@ -6,17 +6,17 @@ coreos: command: start content: | [Unit] - After=opt-kubernetes.mount etcd.service + After=opt-kubernetes.mount etcd2.service ConditionFileIsExecutable=/opt/kubernetes/bin/kube-apiserver Description=Kubernetes API Server Documentation=https://github.com/GoogleCloudPlatform/kubernetes - Requires=opt-kubernetes.mount etcd.service + Requires=opt-kubernetes.mount etcd2.service [Service] ExecStart=/opt/kubernetes/bin/kube-apiserver \ --address=0.0.0.0 \ --port=8080 \ - --etcd-servers=http://127.0.0.1:4001 \ + --etcd-servers=http://127.0.0.1:2379 \ --kubelet-port=10250 \ --service-cluster-ip-range=${SERVICE_CLUSTER_IP_RANGE} Restart=always diff --git a/cluster/libvirt-coreos/user_data_minion.yml b/cluster/libvirt-coreos/user_data_minion.yml index 5c9c5c9d8fe..7c80f9a9d96 100644 --- a/cluster/libvirt-coreos/user_data_minion.yml +++ b/cluster/libvirt-coreos/user_data_minion.yml @@ -6,11 +6,11 @@ coreos: command: start content: | [Unit] - After=opt-kubernetes.mount etcd.service docker.socket + After=opt-kubernetes.mount docker.socket ConditionFileIsExecutable=/opt/kubernetes/bin/kubelet Description=Kubernetes Kubelet Documentation=https://github.com/GoogleCloudPlatform/kubernetes - Requires=opt-kubernetes.mount etcd.service docker.socket + Requires=opt-kubernetes.mount docker.socket [Service] ExecStart=/opt/kubernetes/bin/kubelet \ @@ -29,11 +29,11 @@ coreos: command: start content: | [Unit] - After=opt-kubernetes.mount etcd.service + After=opt-kubernetes.mount ConditionFileIsExecutable=/opt/kubernetes/bin/kube-proxy Description=Kubernetes Proxy Documentation=https://github.com/GoogleCloudPlatform/kubernetes - Requires=opt-kubernetes.mount etcd.service + Requires=opt-kubernetes.mount [Service] ExecStart=/opt/kubernetes/bin/kube-proxy \ diff --git a/cluster/libvirt-coreos/util.sh b/cluster/libvirt-coreos/util.sh index fc3547e7f17..82284b53799 100644 --- a/cluster/libvirt-coreos/util.sh +++ b/cluster/libvirt-coreos/util.sh @@ -183,12 +183,13 @@ function wait-cluster-readiness { function kube-up { detect-master detect-minions + get-kubeconfig-bearertoken initialize-pool keep_base_image initialize-network readonly ssh_keys="$(cat ~/.ssh/id_*.pub | sed 's/^/ - /')" readonly kubernetes_dir="$POOL_PATH/kubernetes" - readonly discovery=$(curl -s https://discovery.etcd.io/new) + readonly discovery=$(curl -s https://discovery.etcd.io/new?size=$(($NUM_MINIONS+1))) readonly machines=$(join , "${KUBE_MINION_IP_ADDRESSES[@]}") From 0535ca92634a045c124201d7750e88346941ba86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9na=C3=AFc=20Huard?= Date: Thu, 13 Aug 2015 11:36:04 +0200 Subject: [PATCH 3/3] Better explain what are the purposes of the libvirt-coreos cluster provider --- docs/getting-started-guides/libvirt-coreos.md | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/docs/getting-started-guides/libvirt-coreos.md b/docs/getting-started-guides/libvirt-coreos.md index c48dd9e4258..e3f56910799 100644 --- a/docs/getting-started-guides/libvirt-coreos.md +++ b/docs/getting-started-guides/libvirt-coreos.md @@ -36,6 +36,7 @@ Getting started with libvirt CoreOS **Table of Contents** - [Highlights](#highlights) +- [Warnings about `libvirt-coreos` use case](#warnings-about-libvirt-coreos-use-case) - [Prerequisites](#prerequisites) - [Setup](#setup) - [Interacting with your Kubernetes cluster with the `kube-*` scripts.](#interacting-with-your-kubernetes-cluster-with-the-kube--scripts) @@ -52,6 +53,30 @@ Getting started with libvirt CoreOS * Reduced disk usage thanks to [COW](https://en.wikibooks.org/wiki/QEMU/Images#Copy_on_write) * Reduced memory footprint thanks to [KSM](https://www.kernel.org/doc/Documentation/vm/ksm.txt) +### Warnings about `libvirt-coreos` use case + +The primary goal of the `libvirt-coreos` cluster provider is to deploy a multi-node Kubernetes cluster on local VMs as fast as possible and to be as light as possible in term of resources used. + +In order to achieve that goal, its deployment is very different from the “standard production deployment” method used on other providers. This was done on purpose in order to implement some optimizations made possible by the fact that we know that all VMs will be running on the same physical machine. + +The `libvirt-coreos` cluster provider doesn’t aim at being production look-alike. + +Another difference is that no security is enforced on `libvirt-coreos` at all. For example, + +* Kube API server is reachable via a clear-text connection (no SSL); +* Kube API server requires no credentials; +* etcd access is not protected; +* Kubernetes secrets are not protected as securely as they are on production environments; +* etc. + +So, an k8s application developer should not validate its interaction with Kubernetes on `libvirt-coreos` because he might technically succeed in doing things that are prohibited on a production environment like: + +* un-authenticated access to Kube API server; +* Access to Kubernetes private data structures inside etcd; +* etc. + +On the other hand, `libvirt-coreos` might be useful for people investigating low level implementation of Kubernetes because debugging techniques like sniffing the network traffic or introspecting the etcd content are easier on `libvirt-coreos` than on a production deployment. + ### Prerequisites 1. Install [dnsmasq](http://www.thekelleys.org.uk/dnsmasq/doc.html)