From 164c4a30f574fe580263b2b5d20ebd7ec1559f27 Mon Sep 17 00:00:00 2001 From: Ian Campbell Date: Wed, 4 Oct 2017 11:13:32 +0100 Subject: [PATCH 1/3] kubernetes: support untainting master via metadata If a stamp file is present in the metadata then untaint. This is useful for dev environments where you only want to start a single vm. The construction of the metadata becomes a little more complex to produce correct json syntax now that there are two (independent) possible options. Likewise the kubelet.sh script now takes the presence of /var/config/kubeadm (rather than /var/config/kubeadm/init) as the signal to use the more structured setup, since we may now have /var/config/kubeadm/untaint-master but not /var/config/kubeadm/init so would otherwise end up passing the contents of `/var/config/userdata` (something like `{ "kubeadm": { "untaint-master": "" } }`) to `kubeadm` and confusing it enormously. Signed-off-by: Ian Campbell --- projects/kubernetes/boot.sh | 13 +++++++++--- .../kubernetes/kubernetes/kubeadm-init.sh | 4 ++++ projects/kubernetes/kubernetes/kubelet.sh | 20 ++++++++++--------- 3 files changed, 25 insertions(+), 12 deletions(-) diff --git a/projects/kubernetes/boot.sh b/projects/kubernetes/boot.sh index 5be099635..241c098c8 100755 --- a/projects/kubernetes/boot.sh +++ b/projects/kubernetes/boot.sh @@ -5,6 +5,7 @@ set -e : ${KUBE_MASTER_VCPUS:=2} : ${KUBE_MASTER_MEM:=1024} : ${KUBE_MASTER_DISK:=4G} +: ${KUBE_MASTER_UNTAINT:=n} : ${KUBE_NODE_VCPUS:=2} : ${KUBE_NODE_MEM:=4096} @@ -27,10 +28,16 @@ if [ $# -eq 0 ] ; then # then we configure for auto init. If it is completely unset then # we do not. if [ -n "${KUBE_MASTER_AUTOINIT+x}" ] ; then - data="{\"kubeadm\": {\"init\": \"${KUBE_MASTER_AUTOINIT}\"} }" - else - data="" + kubeadm_data="${kubeadm_data+$kubeadm_data, }\"init\": \"${KUBE_MASTER_AUTOINIT}\"" fi + if [ "${KUBE_MASTER_UNTAINT}" = "y" ] ; then + kubeadm_data="${kubeadm_data+$kubeadm_data, }\"untaint-master\": \"\"" + fi + + if [ -n "${kubeadm_data}" ] ; then + data="{ \"kubeadm\": { ${kubeadm_data} } }" + fi + state="kube-master-state" : ${KUBE_VCPUS:=$KUBE_MASTER_VCPUS} diff --git a/projects/kubernetes/kubernetes/kubeadm-init.sh b/projects/kubernetes/kubernetes/kubeadm-init.sh index 89403fee1..505f2f5a7 100755 --- a/projects/kubernetes/kubernetes/kubeadm-init.sh +++ b/projects/kubernetes/kubernetes/kubeadm-init.sh @@ -7,3 +7,7 @@ for i in /etc/kubeadm/kube-system.init/*.yaml ; do kubectl create -n kube-system -f "$i" fi done +if [ -f /var/config/kubeadm/untaint-master ] ; then + echo "Removing \"node-role.kubernetes.io/master\" taint from all nodes" + kubectl taint nodes --all node-role.kubernetes.io/master- +fi diff --git a/projects/kubernetes/kubernetes/kubelet.sh b/projects/kubernetes/kubernetes/kubelet.sh index cab7d1f00..c554f120c 100755 --- a/projects/kubernetes/kubernetes/kubelet.sh +++ b/projects/kubernetes/kubernetes/kubelet.sh @@ -15,15 +15,17 @@ await=/etc/kubernetes/kubelet.conf if [ -f "/etc/kubernetes/kubelet.conf" ] ; then echo "kubelet.sh: kubelet already configured" -elif [ -e /var/config/kubeadm/init ] ; then - echo "kubelet.sh: init cluster with metadata \"$(cat /var/config/kubeadm/init)\"" - # This needs to be in the background since it waits for kubelet to start. - # We skip printing the token so it is not persisted in the log. - kubeadm-init.sh --skip-token-print $(cat /var/config/kubeadm/init) & -elif [ -e /var/config/kubeadm/join ] ; then - echo "kubelet.sh: joining cluster with metadata \"$(cat /var/config/kubeadm/join)\"" - kubeadm join --skip-preflight-checks $(cat /var/config/kubeadm/join) - await=/etc/kubernetes/bootstrap-kubelet.conf +elif [ -d /var/config/kubeadm ] ; then + if [ -f /var/config/kubeadm/init ] ; then + echo "kubelet.sh: init cluster with metadata \"$(cat /var/config/kubeadm/init)\"" + # This needs to be in the background since it waits for kubelet to start. + # We skip printing the token so it is not persisted in the log. + kubeadm-init.sh --skip-token-print $(cat /var/config/kubeadm/init) & + elif [ -e /var/config/kubeadm/join ] ; then + echo "kubelet.sh: joining cluster with metadata \"$(cat /var/config/kubeadm/join)\"" + kubeadm join --skip-preflight-checks $(cat /var/config/kubeadm/join) + await=/etc/kubernetes/bootstrap-kubelet.conf + fi elif [ -e /var/config/userdata ] ; then echo "kubelet.sh: joining cluster with metadata \"$(cat /var/config/userdata)\"" kubeadm join --skip-preflight-checks $(cat /var/config/userdata) From 4386cbcdf9b930c7d955fc64eb3e8d0833f8f674 Mon Sep 17 00:00:00 2001 From: Ian Campbell Date: Wed, 4 Oct 2017 12:07:44 +0100 Subject: [PATCH 2/3] kubernetes: bookend kubeadm-init with stamp files. It is possible to get rebooted halfway through the init process, after key files like `/etc/kubernetes/kubelet.conf` have been created but before full cluster setup is complete or networking is applied. Right now the idempotency of kubeadm (or backing out from this half-way state and resuming the initialisation) is not something I have investigated. By dropped stamps before and after at least the situation will be somewhat detectable/diagnosable so the user can e.g. nuke their persistent disk and start again. Signed-off-by: Ian Campbell --- projects/kubernetes/kubernetes/kubeadm-init.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/projects/kubernetes/kubernetes/kubeadm-init.sh b/projects/kubernetes/kubernetes/kubeadm-init.sh index 505f2f5a7..0aa1895f2 100755 --- a/projects/kubernetes/kubernetes/kubeadm-init.sh +++ b/projects/kubernetes/kubernetes/kubeadm-init.sh @@ -1,5 +1,6 @@ #!/bin/sh set -e +touch /var/lib/kubeadm/.kubeadm-init.sh-started kubeadm init --skip-preflight-checks --kubernetes-version @KUBERNETES_VERSION@ $@ for i in /etc/kubeadm/kube-system.init/*.yaml ; do if [ -e "$i" ] ; then @@ -11,3 +12,4 @@ if [ -f /var/config/kubeadm/untaint-master ] ; then echo "Removing \"node-role.kubernetes.io/master\" taint from all nodes" kubectl taint nodes --all node-role.kubernetes.io/master- fi +touch /var/lib/kubeadm/.kubeadm-init.sh-finished From 8d69ed14f4d0e1663752c95c93311cce4e3432d4 Mon Sep 17 00:00:00 2001 From: Ian Campbell Date: Wed, 4 Oct 2017 12:22:17 +0100 Subject: [PATCH 3/3] kubernetes: Update yml Signed-off-by: Ian Campbell --- projects/kubernetes/kube.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/kubernetes/kube.yml b/projects/kubernetes/kube.yml index 88723876a..9db7026f7 100644 --- a/projects/kubernetes/kube.yml +++ b/projects/kubernetes/kube.yml @@ -36,7 +36,7 @@ services: - name: sshd image: linuxkit/sshd:d313eea3d9d7fbcbc927d06a6700325725db2a82 - name: kubelet - image: linuxkitprojects/kubernetes:98d03686d3665b935dcd68da192f79c4cb618ec7 + image: linuxkitprojects/kubernetes:2a42ca12c52a756ffd83ec014f2b396891880e4a files: - path: etc/linuxkit.yml metadata: yaml