diff --git a/cluster/libvirt-coreos/.gitignore b/cluster/libvirt-coreos/.gitignore index 3eb6823b0f2..f26d8e4020a 100644 --- a/cluster/libvirt-coreos/.gitignore +++ b/cluster/libvirt-coreos/.gitignore @@ -1,3 +1,2 @@ /libvirt_storage_pool/ /coreos_production_qemu_image.img.bz2 -/etcd-v2.0.9-linux-amd64.tar.gz diff --git a/cluster/libvirt-coreos/coreos.xml b/cluster/libvirt-coreos/coreos.xml index a0490e2eadd..041f3a01510 100644 --- a/cluster/libvirt-coreos/coreos.xml +++ b/cluster/libvirt-coreos/coreos.xml @@ -35,11 +35,6 @@ - - - - - diff --git a/cluster/libvirt-coreos/user_data.yml b/cluster/libvirt-coreos/user_data.yml index 9df5eeea466..4616c24d44c 100644 --- a/cluster/libvirt-coreos/user_data.yml +++ b/cluster/libvirt-coreos/user_data.yml @@ -14,25 +14,13 @@ write_files: RuntimeMaxUse=50M coreos: - etcd: - name: ${name} - addr: ${public_ip}:4001 - # bind-addr: 0.0.0.0 - peer-addr: ${public_ip}:7001 - # peers: {etcd_peers} + etcd2: discovery: ${discovery} + advertise-client-urls: http://${public_ip}:2379 + initial-advertise-peer-urls: http://${public_ip}:2380 + listen-client-urls: http://0.0.0.0:2379 + listen-peer-urls: http://${public_ip}:2380 units: - - name: etcd.service - drop-ins: - - name: opt-etcd2.conf - content: | - [Unit] - After=opt-etcd.mount - Requires=opt-etcd.mount - - [Service] - ExecStart= - ExecStart=/opt/etcd/bin/etcd - name: static.network command: start content: | @@ -80,7 +68,7 @@ coreos: ExecStart=/usr/sbin/iptables -w -t nat -A POSTROUTING -o eth0 -j MASQUERADE ! -d ${CONTAINER_SUBNET} RemainAfterExit=yes Type=oneshot - - name: etcd.service + - name: etcd2.service command: start - name: docker.service command: start @@ -114,17 +102,6 @@ coreos: Where=/opt/kubernetes Options=ro,trans=virtio,version=9p2000.L Type=9p - - name: opt-etcd.mount - command: start - content: | - [Unit] - ConditionVirtualization=|vm - - [Mount] - What=etcd - Where=/opt/etcd - Options=ro,trans=virtio,version=9p2000.L - Type=9p update: group: ${COREOS_CHANNEL:-alpha} reboot-strategy: off diff --git a/cluster/libvirt-coreos/user_data_master.yml b/cluster/libvirt-coreos/user_data_master.yml index cd48387b278..c34b034fe33 100644 --- a/cluster/libvirt-coreos/user_data_master.yml +++ b/cluster/libvirt-coreos/user_data_master.yml @@ -6,17 +6,17 @@ coreos: command: start content: | [Unit] - After=opt-kubernetes.mount etcd.service + After=opt-kubernetes.mount etcd2.service ConditionFileIsExecutable=/opt/kubernetes/bin/kube-apiserver Description=Kubernetes API Server Documentation=https://github.com/GoogleCloudPlatform/kubernetes - Requires=opt-kubernetes.mount etcd.service + Requires=opt-kubernetes.mount etcd2.service [Service] ExecStart=/opt/kubernetes/bin/kube-apiserver \ --address=0.0.0.0 \ --port=8080 \ - --etcd-servers=http://127.0.0.1:4001 \ + --etcd-servers=http://127.0.0.1:2379 \ --kubelet-port=10250 \ --service-cluster-ip-range=${SERVICE_CLUSTER_IP_RANGE} Restart=always diff --git a/cluster/libvirt-coreos/user_data_minion.yml b/cluster/libvirt-coreos/user_data_minion.yml index 5c9c5c9d8fe..7c80f9a9d96 100644 --- a/cluster/libvirt-coreos/user_data_minion.yml +++ b/cluster/libvirt-coreos/user_data_minion.yml @@ -6,11 +6,11 @@ coreos: command: start content: | [Unit] - After=opt-kubernetes.mount etcd.service docker.socket + After=opt-kubernetes.mount docker.socket ConditionFileIsExecutable=/opt/kubernetes/bin/kubelet Description=Kubernetes Kubelet Documentation=https://github.com/GoogleCloudPlatform/kubernetes - Requires=opt-kubernetes.mount etcd.service docker.socket + Requires=opt-kubernetes.mount docker.socket [Service] ExecStart=/opt/kubernetes/bin/kubelet \ @@ -29,11 +29,11 @@ coreos: command: start content: | [Unit] - After=opt-kubernetes.mount etcd.service + After=opt-kubernetes.mount ConditionFileIsExecutable=/opt/kubernetes/bin/kube-proxy Description=Kubernetes Proxy Documentation=https://github.com/GoogleCloudPlatform/kubernetes - Requires=opt-kubernetes.mount etcd.service + Requires=opt-kubernetes.mount [Service] ExecStart=/opt/kubernetes/bin/kube-proxy \ diff --git a/cluster/libvirt-coreos/util.sh b/cluster/libvirt-coreos/util.sh index 07954c5f4dd..f424d89ddbf 100644 --- a/cluster/libvirt-coreos/util.sh +++ b/cluster/libvirt-coreos/util.sh @@ -26,8 +26,6 @@ export LIBVIRT_DEFAULT_URI=qemu:///system readonly POOL=kubernetes readonly POOL_PATH="$(cd $ROOT && pwd)/libvirt_storage_pool" -ETCD_VERSION=${ETCD_VERSION:-v2.0.9} - # join # Concatenates the list elements with the delimiter passed as first parameter # @@ -96,9 +94,6 @@ function destroy-pool { virsh vol-delete $vol --pool $POOL done - rm -rf "$POOL_PATH"/etcd/* - virsh vol-delete etcd --pool $POOL || true - [[ "$1" == 'keep_base_image' ]] && return set +e @@ -146,18 +141,6 @@ function initialize-pool { render-template "$ROOT/skydns-rc.yaml" > "$POOL_PATH/kubernetes/addons/skydns-rc.yaml" fi - mkdir -p "$POOL_PATH/etcd" - if [[ ! -f "$ROOT/etcd-${ETCD_VERSION}-linux-amd64.tar.gz" ]]; then - wget -P "$ROOT" https://github.com/coreos/etcd/releases/download/${ETCD_VERSION}/etcd-${ETCD_VERSION}-linux-amd64.tar.gz - fi - if [[ "$ROOT/etcd-${ETCD_VERSION}-linux-amd64.tar.gz" -nt "$POOL_PATH/etcd/etcd" ]]; then - tar -x -C "$POOL_PATH/etcd" -f "$ROOT/etcd-${ETCD_VERSION}-linux-amd64.tar.gz" etcd-${ETCD_VERSION}-linux-amd64 - rm -rf "$POOL_PATH/etcd/bin/*" - mkdir -p "$POOL_PATH/etcd/bin" - mv "$POOL_PATH"/etcd/etcd-${ETCD_VERSION}-linux-amd64/{etcd,etcdctl} "$POOL_PATH/etcd/bin" - rm -rf "$POOL_PATH/etcd/etcd-${ETCD_VERSION}-linux-amd64" - fi - virsh pool-refresh $POOL } @@ -200,13 +183,13 @@ function wait-cluster-readiness { function kube-up { detect-master detect-minions + get-kubeconfig-bearertoken initialize-pool keep_base_image initialize-network readonly ssh_keys="$(cat ~/.ssh/id_*.pub | sed 's/^/ - /')" readonly kubernetes_dir="$POOL_PATH/kubernetes" - readonly etcd_dir="$POOL_PATH/etcd" - readonly discovery=$(curl -s https://discovery.etcd.io/new) + readonly discovery=$(curl -s https://discovery.etcd.io/new?size=$(($NUM_MINIONS+1))) readonly machines=$(join , "${KUBE_MINION_IP_ADDRESSES[@]}") diff --git a/docs/getting-started-guides/libvirt-coreos.md b/docs/getting-started-guides/libvirt-coreos.md index c48dd9e4258..e3f56910799 100644 --- a/docs/getting-started-guides/libvirt-coreos.md +++ b/docs/getting-started-guides/libvirt-coreos.md @@ -36,6 +36,7 @@ Getting started with libvirt CoreOS **Table of Contents** - [Highlights](#highlights) +- [Warnings about `libvirt-coreos` use case](#warnings-about-libvirt-coreos-use-case) - [Prerequisites](#prerequisites) - [Setup](#setup) - [Interacting with your Kubernetes cluster with the `kube-*` scripts.](#interacting-with-your-kubernetes-cluster-with-the-kube--scripts) @@ -52,6 +53,30 @@ Getting started with libvirt CoreOS * Reduced disk usage thanks to [COW](https://en.wikibooks.org/wiki/QEMU/Images#Copy_on_write) * Reduced memory footprint thanks to [KSM](https://www.kernel.org/doc/Documentation/vm/ksm.txt) +### Warnings about `libvirt-coreos` use case + +The primary goal of the `libvirt-coreos` cluster provider is to deploy a multi-node Kubernetes cluster on local VMs as fast as possible and to be as light as possible in term of resources used. + +In order to achieve that goal, its deployment is very different from the “standard production deployment” method used on other providers. This was done on purpose in order to implement some optimizations made possible by the fact that we know that all VMs will be running on the same physical machine. + +The `libvirt-coreos` cluster provider doesn’t aim at being production look-alike. + +Another difference is that no security is enforced on `libvirt-coreos` at all. For example, + +* Kube API server is reachable via a clear-text connection (no SSL); +* Kube API server requires no credentials; +* etcd access is not protected; +* Kubernetes secrets are not protected as securely as they are on production environments; +* etc. + +So, an k8s application developer should not validate its interaction with Kubernetes on `libvirt-coreos` because he might technically succeed in doing things that are prohibited on a production environment like: + +* un-authenticated access to Kube API server; +* Access to Kubernetes private data structures inside etcd; +* etc. + +On the other hand, `libvirt-coreos` might be useful for people investigating low level implementation of Kubernetes because debugging techniques like sniffing the network traffic or introspecting the etcd content are easier on `libvirt-coreos` than on a production deployment. + ### Prerequisites 1. Install [dnsmasq](http://www.thekelleys.org.uk/dnsmasq/doc.html)