minor fixes to get a working cluster

2025-08-08 11:38:15 +00:00 · 2015-05-08 16:13:11 -05:00 · 2015-05-08 16:13:11 -05:00 · 1e8ee4d1f1
commit 1e8ee4d1f1
parent 16a448082f
4 changed files with 97 additions and 41 deletions
--- a/cluster/rackspace/cloud-config/master-cloud-config.yaml
+++ b/cluster/rackspace/cloud-config/master-cloud-config.yaml
@ -72,7 +72,7 @@ coreos:
        Type=oneshot
        RemainAfterExit=yes
        ExecStart=/usr/bin/bash /opt/bin/download-release.sh
-    - name: master-apiserver.service
+    - name: kube-apiserver.service
      command: start
      content: |
        [Unit]
@ -84,9 +84,20 @@ coreos:
        Requires=download-release.service
        [Service]
        ExecStartPre=/usr/bin/ln -sf /opt/kubernetes/server/bin/kube-apiserver /opt/bin/kube-apiserver
-        ExecStart=/opt/bin/kube-apiserver --token-auth-file=/var/lib/kubernetes/apiserver/known_tokens.csv --address=127.0.0.1 --port=8080 --etcd_servers=http://127.0.0.1:4001 --portal_net=PORTAL_NET --logtostderr=true --cloud_provider=rackspace --cloud_config=/etc/cloud.conf --v=2
+        ExecStartPre=/usr/bin/mkdir -p /var/lib/kube-apiserver
+        ExecStartPre=/usr/bin/cp /media/configdrive/openstack/content/0000 /var/lib/kube-apiserver/known_tokens.csv
+        ExecStart=/opt/bin/kube-apiserver \
+        --address=127.0.0.1 \
+        --cloud_provider=rackspace \
+        --cloud_config=/etc/cloud.conf \
+        --etcd_servers=http://127.0.0.1:4001 \
+        --logtostderr=true \
+        --port=8080 \
+        --portal_net=PORTAL_NET \
+        --token-auth-file=/var/lib/kube-apiserver/known_tokens.csv \
+        --v=2
        Restart=always
-        RestartSec=2
+        RestartSec=5
    - name: apiserver-advertiser.service
      command: start
      content: |
@ -99,7 +110,7 @@ coreos:
        ExecStart=/bin/sh -c 'etcdctl set /corekube/apiservers/$public_ipv4 $public_ipv4'
        Restart=always
        RestartSec=120
-    - name: master-controller-manager.service
+    - name: kube-controller-manager.service
      command: start
      content: |
        [Unit]
@ -107,14 +118,19 @@ coreos:
        Documentation=https://github.com/GoogleCloudPlatform/kubernetes
        After=network-online.target
        Requires=network-online.target
-        After=master-apiserver.service
-        Requires=master-apiserver.service
+        After=kube-apiserver.service
+        Requires=kube-apiserver.service
        [Service]
        ExecStartPre=/usr/bin/ln -sf /opt/kubernetes/server/bin/kube-controller-manager /opt/bin/kube-controller-manager
-        ExecStart=/opt/bin/kube-controller-manager --master=127.0.0.1:8080 --logtostderr=true --cloud_provider=rackspace --cloud_config=/etc/cloud.conf --v=2
+        ExecStart=/opt/bin/kube-controller-manager \
+        --cloud_provider=rackspace \
+        --cloud_config=/etc/cloud.conf \
+        --logtostderr=true \
+        --master=127.0.0.1:8080 \
+        --v=2
        Restart=always
-        RestartSec=2
-    - name: master-scheduler.service
+        RestartSec=5
+    - name: kube-scheduler.service
      command: start
      content: |
        [Unit]
@ -122,27 +138,33 @@ coreos:
        Documentation=https://github.com/GoogleCloudPlatform/kubernetes
        After=network-online.target
        Requires=network-online.target
-        After=master-apiserver.service
-        Requires=master-apiserver.service
+        After=kube-apiserver.service
+        Requires=kube-apiserver.service
        [Service]
        ExecStartPre=/usr/bin/ln -sf /opt/kubernetes/server/bin/kube-scheduler /opt/bin/kube-scheduler
-        ExecStart=/opt/bin/kube-scheduler --master=127.0.0.1:8080 --logtostderr=true
+        ExecStart=/opt/bin/kube-scheduler \
+        --logtostderr=true \
+        --master=127.0.0.1:8080
        Restart=always
-        RestartSec=10
-    - name: master-register.service
+        RestartSec=5
+    - name: kube-register.service
      command: start
      content: |
        [Unit]
        Description=Kubernetes Registration Service
        Documentation=https://github.com/kelseyhightower/kube-register
-
+        Requires=kube-apiserver.service
+        After=kube-apiserver.service
+        Requires=fleet.service
+        After=fleet.service
        [Service]
-        ExecStartPre=/usr/bin/wget -N -P /opt/bin https://storage.googleapis.com/kubernetes/kube-register
+        ExecStartPre=/usr/bin/curl -L -o /opt/bin/kube-register -z /opt/bin/kube-register https://github.com/kelseyhightower/kube-register/releases/download/v0.0.3/kube-register-0.0.3-linux-amd64
        ExecStartPre=/usr/bin/chmod +x /opt/bin/kube-register
        ExecStart=/opt/bin/kube-register \
-        --metadata=kubernetes_role=minion \
+        --api-endpoint=http://127.0.0.1:8080 \
        --fleet-endpoint=unix:///var/run/fleet.sock \
-        --api-endpoint=http://127.0.0.1:8080
+        --healthz-port=10248 \
+        --metadata=kubernetes_role=minion
        Restart=always
        RestartSec=10
    #Running nginx service with --net="host" is a necessary evil until running all k8s services in docker.
--- a/cluster/rackspace/cloud-config/minion-cloud-config.yaml
+++ b/cluster/rackspace/cloud-config/minion-cloud-config.yaml
@ -8,7 +8,7 @@ write_files:
      m=$(echo $(etcdctl ls --recursive /corekube/apiservers | cut -d/ -f4 | sort) | tr ' ' ,)
      mkdir -p /run/kubelet
      echo "APISERVER_IPS=$m" > /run/kubelet/apiservers.env
-      echo "FIRST_APISERVER_URL=http://${m%%\,*}:6443" >> /run/kubelet/apiservers.env
+      echo "FIRST_APISERVER_URL=https://${m%%\,*}:6443" >> /run/kubelet/apiservers.env
  - path: /opt/bin/download-release.sh
    permissions: 0755
    content: |
@ -26,6 +26,37 @@ write_files:
      else
        echo "kubernetes release found. Skipping download."
      fi
+  - path: /run/setup-auth.sh
+    permissions: 0755
+    content: |
+      #!/bin/bash -e
+      set -x
+      /usr/bin/mkdir -p /var/lib/kubelet
+      /bin/echo  "{\"BearerToken\": \"KUBE_BEARER_TOKEN\", \"Insecure\": true }" > /var/lib/kubelet/kubernetes_auth
+  - path: /run/config-kube-proxy.sh
+    permissions: 0755
+    content: |
+      #!/bin/bash -e
+      set -x
+      /usr/bin/mkdir -p /var/lib/kube-proxy
+      cat > /var/lib/kube-proxy/kubeconfig << EOF
+      apiVersion: v1
+      kind: Config
+      users:
+      - name: kube-proxy
+        user:
+          token: KUBE_PROXY_TOKEN
+      clusters:
+      - name: local
+        cluster:
+          insecure-skip-tls-verify: true
+      contexts:
+      - context:
+          cluster: local
+          user: kube-proxy
+        name: service-account-context
+      current-context: service-account-context
+      EOF

 coreos:
  etcd:
@ -53,7 +84,7 @@ coreos:
      - name: 50-network-config.conf
        content: |
          [Service]
-          ExecStartPre=/usr/bin/etcdctl set /coreos.com/network/config '{ "Network": "10.240.0.0/16", "Backend": {"Type": "host-gw"}}'
+          ExecStartPre=/usr/bin/etcdctl set /coreos.com/network/config '{ "Network": "KUBE_NETWORK", "Backend": {"Type": "host-gw"}}'
          ExecStart=
          ExecStart=/usr/libexec/sdnotify-proxy /run/flannel/sd.sock \
            /usr/bin/docker run --net=host --privileged=true --rm \
@ -83,7 +114,7 @@ coreos:
        Type=oneshot
        RemainAfterExit=yes
        ExecStart=/usr/bin/bash /opt/bin/download-release.sh
-    - name: minion-kubelet.service
+    - name: kubelet.service
      command: start
      content: |
        [Unit]
@ -97,11 +128,20 @@ coreos:
        Requires=download-release.service
        [Service]
        EnvironmentFile=/run/kubelet/apiservers.env
+        ExecStartPre=/run/setup-auth.sh
        ExecStartPre=/usr/bin/ln -sf /opt/kubernetes/server/bin/kubelet /opt/bin/kubelet
-        ExecStart=/opt/bin/kubelet --address=$private_ipv4 --hostname_override=$private_ipv4 --api_servers=${FIRST_APISERVER_URL} --logtostderr=true --cluster_dns=DNS_SERVER_IP --cluster_domain=DNS_DOMAIN
+        ExecStart=/opt/bin/kubelet \
+        --address=$private_ipv4 \
+        --api_servers=${FIRST_APISERVER_URL} \
+        --cluster_dns=DNS_SERVER_IP \
+        --cluster_domain=DNS_DOMAIN \
+        --healthz-bind-address=$private_ipv4 \
+        --hostname_override=$private_ipv4 \
+        --logtostderr=true \
+        --v=2
        Restart=always
-        RestartSec=2
-    - name: minion-proxy.service
+        RestartSec=5
+    - name: kube-proxy.service
      command: start
      content: |
        [Unit]
@ -115,10 +155,15 @@ coreos:
        Requires=download-release.service
        [Service]
        EnvironmentFile=/run/kubelet/apiservers.env
+        ExecStartPre=/run/config-kube-proxy.sh
        ExecStartPre=/usr/bin/ln -sf /opt/kubernetes/server/bin/kube-proxy /opt/bin/kube-proxy
-        ExecStart=/opt/bin/kube-proxy --bind_address=$private_ipv4 --master=${FIRST_APISERVER_URL} --logtostderr=true
+        ExecStart=/opt/bin/kube-proxy \
+        --bind_address=$private_ipv4 \
+        --kubeconfig=/var/lib/kube-proxy/kubeconfig \
+        --logtostderr=true \
+        --master=${FIRST_APISERVER_URL}
        Restart=always
-        RestartSec=2
+        RestartSec=5
    - name: kubelet-sighup.path
      command: start
      content: |
@ -139,18 +184,6 @@ coreos:
      content: |
        [Service]
        ExecStart=/usr/bin/pkill -SIGHUP -f kube-proxy
-    - name: minion-advertiser.service
-      command: start
-      content: |
-        [Unit]
-        Description=Kubernetes Minion Advertiser
-        After=etcd.service
-        Requires=etcd.service
-        After=minion-kubelet.service
-        [Service]
-        ExecStart=/bin/sh -c 'while :; do etcdctl set /corekube/minions/$private_ipv4 $private_ipv4 --ttl 300; sleep 120; done'
-        Restart=always
-        RestartSec=120
    - name: apiserver-finder.service
      command: start
      content: |
--- a/cluster/rackspace/config-default.sh
+++ b/cluster/rackspace/config-default.sh
@ -35,7 +35,7 @@ KUBE_MINION_FLAVOR="${KUBE_MINION_FLAVOR-performance1-2}"
 RAX_NUM_MINIONS="${RAX_NUM_MINIONS-4}"
 MINION_TAG="tags=${INSTANCE_PREFIX}-minion"
 MINION_NAMES=($(eval echo ${INSTANCE_PREFIX}-minion-{1..${RAX_NUM_MINIONS}}))
-KUBE_NETWORK=($(eval echo "10.240.{1..${RAX_NUM_MINIONS}}.0/24"))
+KUBE_NETWORK="10.240.0.0/16"
 PORTAL_NET="10.0.0.0/16"

 # Optional: Install node monitoring.
--- a/cluster/rackspace/util.sh
+++ b/cluster/rackspace/util.sh
@ -137,7 +137,7 @@ copy_dev_tarballs() {
 prep_known_tokens() {
  for (( i=0; i<${#MINION_NAMES[@]}; i++)); do
    generate_kubelet_tokens ${MINION_NAMES[i]}
-    cat ${KUBE_TEMP}/${MINION_NAMES[i]}_token.csv >> ${KUBE_TEMP}/known_tokens.csv
+    cat ${KUBE_TEMP}/${MINION_NAMES[i]}_tokens.csv >> ${KUBE_TEMP}/known_tokens.csv
  done

    # Generate tokens for other "service accounts".  Append to known_tokens.
@ -180,7 +180,7 @@ rax-boot-master() {
 --meta ${MASTER_TAG} \
 --meta ETCD=${DISCOVERY_ID} \
 --user-data ${KUBE_TEMP}/master-cloud-config.yaml \
--file /var/lib/kubernetes/apiserver/known_tokens.csv=${KUBE_TEMP}/known_tokens.csv \
+--file /var/lib/kube-apiserver/known_tokens.csv=${KUBE_TEMP}/known_tokens.csv \
 --config-drive true \
 --nic net-id=${NETWORK_UUID} \
 ${MASTER_NAME}"
@ -208,6 +208,7 @@ rax-boot-minions() {
        -e "s|ENABLE_NODE_LOGGING|${ENABLE_NODE_LOGGING:-false}|" \
        -e "s|INDEX|$((i + 1))|g" \
        -e "s|KUBE_BEARER_TOKEN|${KUBE_BEARER_TOKEN}|" \
+        -e "s|KUBE_NETWORK|${KUBE_NETWORK}|" \
        -e "s|KUBE_PROXY_TOKEN|${KUBE_PROXY_TOKEN}|" \
        -e "s|LOGGING_DESTINATION|${LOGGING_DESTINATION:-}|" \
    $(dirname $0)/rackspace/cloud-config/minion-cloud-config.yaml > $KUBE_TEMP/minion-cloud-config-$(($i + 1)).yaml