From 1e8ee4d1f1e5ec52aab91f197e66027952021fde Mon Sep 17 00:00:00 2001 From: Ryan Richard Date: Fri, 8 May 2015 16:13:11 -0500 Subject: [PATCH] minor fixes to get a working cluster --- .../cloud-config/master-cloud-config.yaml | 58 ++++++++++----- .../cloud-config/minion-cloud-config.yaml | 73 ++++++++++++++----- cluster/rackspace/config-default.sh | 2 +- cluster/rackspace/util.sh | 5 +- 4 files changed, 97 insertions(+), 41 deletions(-) diff --git a/cluster/rackspace/cloud-config/master-cloud-config.yaml b/cluster/rackspace/cloud-config/master-cloud-config.yaml index e23f0e5d64b..83e09db30b7 100644 --- a/cluster/rackspace/cloud-config/master-cloud-config.yaml +++ b/cluster/rackspace/cloud-config/master-cloud-config.yaml @@ -72,7 +72,7 @@ coreos: Type=oneshot RemainAfterExit=yes ExecStart=/usr/bin/bash /opt/bin/download-release.sh - - name: master-apiserver.service + - name: kube-apiserver.service command: start content: | [Unit] @@ -84,9 +84,20 @@ coreos: Requires=download-release.service [Service] ExecStartPre=/usr/bin/ln -sf /opt/kubernetes/server/bin/kube-apiserver /opt/bin/kube-apiserver - ExecStart=/opt/bin/kube-apiserver --token-auth-file=/var/lib/kubernetes/apiserver/known_tokens.csv --address=127.0.0.1 --port=8080 --etcd_servers=http://127.0.0.1:4001 --portal_net=PORTAL_NET --logtostderr=true --cloud_provider=rackspace --cloud_config=/etc/cloud.conf --v=2 + ExecStartPre=/usr/bin/mkdir -p /var/lib/kube-apiserver + ExecStartPre=/usr/bin/cp /media/configdrive/openstack/content/0000 /var/lib/kube-apiserver/known_tokens.csv + ExecStart=/opt/bin/kube-apiserver \ + --address=127.0.0.1 \ + --cloud_provider=rackspace \ + --cloud_config=/etc/cloud.conf \ + --etcd_servers=http://127.0.0.1:4001 \ + --logtostderr=true \ + --port=8080 \ + --portal_net=PORTAL_NET \ + --token-auth-file=/var/lib/kube-apiserver/known_tokens.csv \ + --v=2 Restart=always - RestartSec=2 + RestartSec=5 - name: apiserver-advertiser.service command: start content: | @@ -99,7 +110,7 @@ coreos: ExecStart=/bin/sh -c 'etcdctl set /corekube/apiservers/$public_ipv4 $public_ipv4' Restart=always RestartSec=120 - - name: master-controller-manager.service + - name: kube-controller-manager.service command: start content: | [Unit] @@ -107,14 +118,19 @@ coreos: Documentation=https://github.com/GoogleCloudPlatform/kubernetes After=network-online.target Requires=network-online.target - After=master-apiserver.service - Requires=master-apiserver.service + After=kube-apiserver.service + Requires=kube-apiserver.service [Service] ExecStartPre=/usr/bin/ln -sf /opt/kubernetes/server/bin/kube-controller-manager /opt/bin/kube-controller-manager - ExecStart=/opt/bin/kube-controller-manager --master=127.0.0.1:8080 --logtostderr=true --cloud_provider=rackspace --cloud_config=/etc/cloud.conf --v=2 + ExecStart=/opt/bin/kube-controller-manager \ + --cloud_provider=rackspace \ + --cloud_config=/etc/cloud.conf \ + --logtostderr=true \ + --master=127.0.0.1:8080 \ + --v=2 Restart=always - RestartSec=2 - - name: master-scheduler.service + RestartSec=5 + - name: kube-scheduler.service command: start content: | [Unit] @@ -122,27 +138,33 @@ coreos: Documentation=https://github.com/GoogleCloudPlatform/kubernetes After=network-online.target Requires=network-online.target - After=master-apiserver.service - Requires=master-apiserver.service + After=kube-apiserver.service + Requires=kube-apiserver.service [Service] ExecStartPre=/usr/bin/ln -sf /opt/kubernetes/server/bin/kube-scheduler /opt/bin/kube-scheduler - ExecStart=/opt/bin/kube-scheduler --master=127.0.0.1:8080 --logtostderr=true + ExecStart=/opt/bin/kube-scheduler \ + --logtostderr=true \ + --master=127.0.0.1:8080 Restart=always - RestartSec=10 - - name: master-register.service + RestartSec=5 + - name: kube-register.service command: start content: | [Unit] Description=Kubernetes Registration Service Documentation=https://github.com/kelseyhightower/kube-register - + Requires=kube-apiserver.service + After=kube-apiserver.service + Requires=fleet.service + After=fleet.service [Service] - ExecStartPre=/usr/bin/wget -N -P /opt/bin https://storage.googleapis.com/kubernetes/kube-register + ExecStartPre=/usr/bin/curl -L -o /opt/bin/kube-register -z /opt/bin/kube-register https://github.com/kelseyhightower/kube-register/releases/download/v0.0.3/kube-register-0.0.3-linux-amd64 ExecStartPre=/usr/bin/chmod +x /opt/bin/kube-register ExecStart=/opt/bin/kube-register \ - --metadata=kubernetes_role=minion \ + --api-endpoint=http://127.0.0.1:8080 \ --fleet-endpoint=unix:///var/run/fleet.sock \ - --api-endpoint=http://127.0.0.1:8080 + --healthz-port=10248 \ + --metadata=kubernetes_role=minion Restart=always RestartSec=10 #Running nginx service with --net="host" is a necessary evil until running all k8s services in docker. diff --git a/cluster/rackspace/cloud-config/minion-cloud-config.yaml b/cluster/rackspace/cloud-config/minion-cloud-config.yaml index d01fd1f8137..dce60807b20 100644 --- a/cluster/rackspace/cloud-config/minion-cloud-config.yaml +++ b/cluster/rackspace/cloud-config/minion-cloud-config.yaml @@ -8,7 +8,7 @@ write_files: m=$(echo $(etcdctl ls --recursive /corekube/apiservers | cut -d/ -f4 | sort) | tr ' ' ,) mkdir -p /run/kubelet echo "APISERVER_IPS=$m" > /run/kubelet/apiservers.env - echo "FIRST_APISERVER_URL=http://${m%%\,*}:6443" >> /run/kubelet/apiservers.env + echo "FIRST_APISERVER_URL=https://${m%%\,*}:6443" >> /run/kubelet/apiservers.env - path: /opt/bin/download-release.sh permissions: 0755 content: | @@ -26,6 +26,37 @@ write_files: else echo "kubernetes release found. Skipping download." fi + - path: /run/setup-auth.sh + permissions: 0755 + content: | + #!/bin/bash -e + set -x + /usr/bin/mkdir -p /var/lib/kubelet + /bin/echo "{\"BearerToken\": \"KUBE_BEARER_TOKEN\", \"Insecure\": true }" > /var/lib/kubelet/kubernetes_auth + - path: /run/config-kube-proxy.sh + permissions: 0755 + content: | + #!/bin/bash -e + set -x + /usr/bin/mkdir -p /var/lib/kube-proxy + cat > /var/lib/kube-proxy/kubeconfig << EOF + apiVersion: v1 + kind: Config + users: + - name: kube-proxy + user: + token: KUBE_PROXY_TOKEN + clusters: + - name: local + cluster: + insecure-skip-tls-verify: true + contexts: + - context: + cluster: local + user: kube-proxy + name: service-account-context + current-context: service-account-context + EOF coreos: etcd: @@ -53,7 +84,7 @@ coreos: - name: 50-network-config.conf content: | [Service] - ExecStartPre=/usr/bin/etcdctl set /coreos.com/network/config '{ "Network": "10.240.0.0/16", "Backend": {"Type": "host-gw"}}' + ExecStartPre=/usr/bin/etcdctl set /coreos.com/network/config '{ "Network": "KUBE_NETWORK", "Backend": {"Type": "host-gw"}}' ExecStart= ExecStart=/usr/libexec/sdnotify-proxy /run/flannel/sd.sock \ /usr/bin/docker run --net=host --privileged=true --rm \ @@ -83,7 +114,7 @@ coreos: Type=oneshot RemainAfterExit=yes ExecStart=/usr/bin/bash /opt/bin/download-release.sh - - name: minion-kubelet.service + - name: kubelet.service command: start content: | [Unit] @@ -97,11 +128,20 @@ coreos: Requires=download-release.service [Service] EnvironmentFile=/run/kubelet/apiservers.env + ExecStartPre=/run/setup-auth.sh ExecStartPre=/usr/bin/ln -sf /opt/kubernetes/server/bin/kubelet /opt/bin/kubelet - ExecStart=/opt/bin/kubelet --address=$private_ipv4 --hostname_override=$private_ipv4 --api_servers=${FIRST_APISERVER_URL} --logtostderr=true --cluster_dns=DNS_SERVER_IP --cluster_domain=DNS_DOMAIN + ExecStart=/opt/bin/kubelet \ + --address=$private_ipv4 \ + --api_servers=${FIRST_APISERVER_URL} \ + --cluster_dns=DNS_SERVER_IP \ + --cluster_domain=DNS_DOMAIN \ + --healthz-bind-address=$private_ipv4 \ + --hostname_override=$private_ipv4 \ + --logtostderr=true \ + --v=2 Restart=always - RestartSec=2 - - name: minion-proxy.service + RestartSec=5 + - name: kube-proxy.service command: start content: | [Unit] @@ -115,10 +155,15 @@ coreos: Requires=download-release.service [Service] EnvironmentFile=/run/kubelet/apiservers.env + ExecStartPre=/run/config-kube-proxy.sh ExecStartPre=/usr/bin/ln -sf /opt/kubernetes/server/bin/kube-proxy /opt/bin/kube-proxy - ExecStart=/opt/bin/kube-proxy --bind_address=$private_ipv4 --master=${FIRST_APISERVER_URL} --logtostderr=true + ExecStart=/opt/bin/kube-proxy \ + --bind_address=$private_ipv4 \ + --kubeconfig=/var/lib/kube-proxy/kubeconfig \ + --logtostderr=true \ + --master=${FIRST_APISERVER_URL} Restart=always - RestartSec=2 + RestartSec=5 - name: kubelet-sighup.path command: start content: | @@ -139,18 +184,6 @@ coreos: content: | [Service] ExecStart=/usr/bin/pkill -SIGHUP -f kube-proxy - - name: minion-advertiser.service - command: start - content: | - [Unit] - Description=Kubernetes Minion Advertiser - After=etcd.service - Requires=etcd.service - After=minion-kubelet.service - [Service] - ExecStart=/bin/sh -c 'while :; do etcdctl set /corekube/minions/$private_ipv4 $private_ipv4 --ttl 300; sleep 120; done' - Restart=always - RestartSec=120 - name: apiserver-finder.service command: start content: | diff --git a/cluster/rackspace/config-default.sh b/cluster/rackspace/config-default.sh index aa6be717f2a..6cc4fc918aa 100644 --- a/cluster/rackspace/config-default.sh +++ b/cluster/rackspace/config-default.sh @@ -35,7 +35,7 @@ KUBE_MINION_FLAVOR="${KUBE_MINION_FLAVOR-performance1-2}" RAX_NUM_MINIONS="${RAX_NUM_MINIONS-4}" MINION_TAG="tags=${INSTANCE_PREFIX}-minion" MINION_NAMES=($(eval echo ${INSTANCE_PREFIX}-minion-{1..${RAX_NUM_MINIONS}})) -KUBE_NETWORK=($(eval echo "10.240.{1..${RAX_NUM_MINIONS}}.0/24")) +KUBE_NETWORK="10.240.0.0/16" PORTAL_NET="10.0.0.0/16" # Optional: Install node monitoring. diff --git a/cluster/rackspace/util.sh b/cluster/rackspace/util.sh index e9caa8a8e31..a1490ee2234 100644 --- a/cluster/rackspace/util.sh +++ b/cluster/rackspace/util.sh @@ -137,7 +137,7 @@ copy_dev_tarballs() { prep_known_tokens() { for (( i=0; i<${#MINION_NAMES[@]}; i++)); do generate_kubelet_tokens ${MINION_NAMES[i]} - cat ${KUBE_TEMP}/${MINION_NAMES[i]}_token.csv >> ${KUBE_TEMP}/known_tokens.csv + cat ${KUBE_TEMP}/${MINION_NAMES[i]}_tokens.csv >> ${KUBE_TEMP}/known_tokens.csv done # Generate tokens for other "service accounts". Append to known_tokens. @@ -180,7 +180,7 @@ rax-boot-master() { --meta ${MASTER_TAG} \ --meta ETCD=${DISCOVERY_ID} \ --user-data ${KUBE_TEMP}/master-cloud-config.yaml \ ---file /var/lib/kubernetes/apiserver/known_tokens.csv=${KUBE_TEMP}/known_tokens.csv \ +--file /var/lib/kube-apiserver/known_tokens.csv=${KUBE_TEMP}/known_tokens.csv \ --config-drive true \ --nic net-id=${NETWORK_UUID} \ ${MASTER_NAME}" @@ -208,6 +208,7 @@ rax-boot-minions() { -e "s|ENABLE_NODE_LOGGING|${ENABLE_NODE_LOGGING:-false}|" \ -e "s|INDEX|$((i + 1))|g" \ -e "s|KUBE_BEARER_TOKEN|${KUBE_BEARER_TOKEN}|" \ + -e "s|KUBE_NETWORK|${KUBE_NETWORK}|" \ -e "s|KUBE_PROXY_TOKEN|${KUBE_PROXY_TOKEN}|" \ -e "s|LOGGING_DESTINATION|${LOGGING_DESTINATION:-}|" \ $(dirname $0)/rackspace/cloud-config/minion-cloud-config.yaml > $KUBE_TEMP/minion-cloud-config-$(($i + 1)).yaml