diff --git a/test/kubemark/gce/util.sh b/test/kubemark/gce/util.sh index fa65061b3ef..8e96268676d 100644 --- a/test/kubemark/gce/util.sh +++ b/test/kubemark/gce/util.sh @@ -76,6 +76,9 @@ function create-master-instance-with-resources { --scopes "storage-ro,compute-rw,logging-write" \ --boot-disk-size "${MASTER_ROOT_DISK_SIZE}" \ --disk "name=${MASTER_NAME}-pd,device-name=master-pd,mode=rw,boot=no,auto-delete=no" + + run-gcloud-compute-with-retries instances add-metadata "${MASTER_NAME}" \ + --metadata-from-file startup-script="${KUBE_ROOT}/test/kubemark/resources/start-kubemark-master.sh" if [ "${EVENT_PD:-false}" == "true" ]; then echo "Attaching ${MASTER_NAME}-event-pd to ${MASTER_NAME}" diff --git a/test/kubemark/resources/start-kubemark-master.sh b/test/kubemark/resources/start-kubemark-master.sh index 84914964069..9e4340390a7 100644 --- a/test/kubemark/resources/start-kubemark-master.sh +++ b/test/kubemark/resources/start-kubemark-master.sh @@ -138,8 +138,8 @@ function mount-pd() { # Create kubeconfig for controller-manager's service account authentication. function create-kubecontrollermanager-kubeconfig { echo "Creating kube-controller-manager kubeconfig file" - mkdir -p /etc/srv/kubernetes/kube-controller-manager - cat </etc/srv/kubernetes/kube-controller-manager/kubeconfig + mkdir -p "${KUBE_ROOT}/k8s_auth_data/kube-controller-manager" + cat <"${KUBE_ROOT}/k8s_auth_data/kube-controller-manager/kubeconfig" apiVersion: v1 kind: Config users: @@ -162,8 +162,8 @@ EOF function create-kubescheduler-kubeconfig { echo "Creating kube-scheduler kubeconfig file" - mkdir -p /etc/srv/kubernetes/kube-scheduler - cat </etc/srv/kubernetes/kube-scheduler/kubeconfig + mkdir -p "${KUBE_ROOT}/k8s_auth_data/kube-scheduler" + cat <"${KUBE_ROOT}/k8s_auth_data/kube-scheduler/kubeconfig" apiVersion: v1 kind: Config users: @@ -419,7 +419,9 @@ echo "Start to configure master instance for kubemark" # Extract files from the server tar and setup master env variables. cd "${KUBE_ROOT}" -tar xzf kubernetes-server-linux-amd64.tar.gz +if [[ ! -d "${KUBE_ROOT}/kubernetes" ]]; then + tar xzf kubernetes-server-linux-amd64.tar.gz +fi source "${KUBE_ROOT}/kubemark-master-env.sh" # Setup IP firewall rules, required directory structure and etcd config. @@ -429,15 +431,18 @@ setup-kubelet-dir delete-default-etcd-configs compute-etcd-variables -# Setup authentication token and kubeconfig for kube-controller-manager. -KUBE_CONTROLLER_MANAGER_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null) -echo "${KUBE_CONTROLLER_MANAGER_TOKEN},system:kube-controller-manager,uid:system:kube-controller-manager" >> /etc/srv/kubernetes/known_tokens.csv -create-kubecontrollermanager-kubeconfig - -# Setup authentication token and kubeconfig for kube-scheduler. -KUBE_SCHEDULER_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null) -echo "${KUBE_SCHEDULER_TOKEN},system:kube-scheduler,uid:system:kube-scheduler" >> /etc/srv/kubernetes/known_tokens.csv -create-kubescheduler-kubeconfig +# Setup authentication tokens and kubeconfigs for kube-controller-manager and kube-scheduler, +# only if their kubeconfigs don't already exist as this script could be running on reboot. +if [[ ! -f "${KUBE_ROOT}/k8s_auth_data/kube-controller-manager/kubeconfig" ]]; then + KUBE_CONTROLLER_MANAGER_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null) + echo "${KUBE_CONTROLLER_MANAGER_TOKEN},system:kube-controller-manager,uid:system:kube-controller-manager" >> "${KUBE_ROOT}/k8s_auth_data/known_tokens.csv" + create-kubecontrollermanager-kubeconfig +fi +if [[ ! -f "${KUBE_ROOT}/k8s_auth_data/kube-scheduler/kubeconfig" ]]; then + KUBE_SCHEDULER_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null) + echo "${KUBE_SCHEDULER_TOKEN},system:kube-scheduler,uid:system:kube-scheduler" >> "${KUBE_ROOT}/k8s_auth_data/known_tokens.csv" + create-kubescheduler-kubeconfig +fi # Mount master PD for etcd and create symbolic links to it. { @@ -447,9 +452,13 @@ create-kubescheduler-kubeconfig mkdir -m 700 -p "${main_etcd_mount_point}/var/etcd" ln -s -f "${main_etcd_mount_point}/var/etcd" /var/etcd mkdir -p /etc/srv - # Contains the dynamically generated apiserver auth certs and keys. + # Setup the dynamically generated apiserver auth certs and keys to pd. mkdir -p "${main_etcd_mount_point}/srv/kubernetes" ln -s -f "${main_etcd_mount_point}/srv/kubernetes" /etc/srv/kubernetes + # Copy the files to the PD only if they don't exist (so we do it only the first time). + if [[ "$(ls -A {main_etcd_mount_point}/srv/kubernetes/)" == "" ]]; then + cp -r "${KUBE_ROOT}"/k8s_auth_data/* "${main_etcd_mount_point}/srv/kubernetes/" + fi # Directory for kube-apiserver to store SSH key (if necessary). mkdir -p "${main_etcd_mount_point}/srv/sshproxy" ln -s -f "${main_etcd_mount_point}/srv/sshproxy" /etc/srv/sshproxy diff --git a/test/kubemark/start-kubemark.sh b/test/kubemark/start-kubemark.sh index 08beb33ee55..e1615b88037 100755 --- a/test/kubemark/start-kubemark.sh +++ b/test/kubemark/start-kubemark.sh @@ -94,18 +94,18 @@ function wait-for-master-reachability { # Write all the relevant certs/keys/tokens to the master. function write-pki-config-to-master { - PKI_SETUP_CMD="sudo mkdir /home/kubernetes -p && sudo mkdir /etc/srv/kubernetes -p && \ - sudo bash -c \"echo ${CA_CERT_BASE64} | base64 --decode > /etc/srv/kubernetes/ca.crt\" && \ - sudo bash -c \"echo ${MASTER_CERT_BASE64} | base64 --decode > /etc/srv/kubernetes/server.cert\" && \ - sudo bash -c \"echo ${MASTER_KEY_BASE64} | base64 --decode > /etc/srv/kubernetes/server.key\" && \ - sudo bash -c \"echo ${KUBECFG_CERT_BASE64} | base64 --decode > /etc/srv/kubernetes/kubecfg.crt\" && \ - sudo bash -c \"echo ${KUBECFG_KEY_BASE64} | base64 --decode > /etc/srv/kubernetes/kubecfg.key\" && \ - sudo bash -c \"echo \"${KUBE_BEARER_TOKEN},admin,admin\" > /etc/srv/kubernetes/known_tokens.csv\" && \ - sudo bash -c \"echo \"${KUBELET_TOKEN},kubelet,uid:kubelet,system:nodes\" >> /etc/srv/kubernetes/known_tokens.csv\" && \ - sudo bash -c \"echo \"${KUBE_PROXY_TOKEN},system:kube-proxy,uid:kube_proxy\" >> /etc/srv/kubernetes/known_tokens.csv\" && \ - sudo bash -c \"echo \"${HEAPSTER_TOKEN},system:heapster,uid:heapster\" >> /etc/srv/kubernetes/known_tokens.csv\" && \ - sudo bash -c \"echo \"${NODE_PROBLEM_DETECTOR_TOKEN},system:node-problem-detector,uid:system:node-problem-detector\" >> /etc/srv/kubernetes/known_tokens.csv\" && \ - sudo bash -c \"echo ${KUBE_PASSWORD},admin,admin > /etc/srv/kubernetes/basic_auth.csv\"" + PKI_SETUP_CMD="sudo mkdir /home/kubernetes/k8s_auth_data -p && \ + sudo bash -c \"echo ${CA_CERT_BASE64} | base64 --decode > /home/kubernetes/k8s_auth_data/ca.crt\" && \ + sudo bash -c \"echo ${MASTER_CERT_BASE64} | base64 --decode > /home/kubernetes/k8s_auth_data/server.cert\" && \ + sudo bash -c \"echo ${MASTER_KEY_BASE64} | base64 --decode > /home/kubernetes/k8s_auth_data/server.key\" && \ + sudo bash -c \"echo ${KUBECFG_CERT_BASE64} | base64 --decode > /home/kubernetes/k8s_auth_data/kubecfg.crt\" && \ + sudo bash -c \"echo ${KUBECFG_KEY_BASE64} | base64 --decode > /home/kubernetes/k8s_auth_data/kubecfg.key\" && \ + sudo bash -c \"echo \"${KUBE_BEARER_TOKEN},admin,admin\" > /home/kubernetes/k8s_auth_data/known_tokens.csv\" && \ + sudo bash -c \"echo \"${KUBELET_TOKEN},system:node:node-name,uid:kubelet,system:nodes\" >> /home/kubernetes/k8s_auth_data/known_tokens.csv\" && \ + sudo bash -c \"echo \"${KUBE_PROXY_TOKEN},system:kube-proxy,uid:kube_proxy\" >> /home/kubernetes/k8s_auth_data/known_tokens.csv\" && \ + sudo bash -c \"echo \"${HEAPSTER_TOKEN},system:heapster,uid:heapster\" >> /home/kubernetes/k8s_auth_data/known_tokens.csv\" && \ + sudo bash -c \"echo \"${NODE_PROBLEM_DETECTOR_TOKEN},system:node-problem-detector,uid:system:node-problem-detector\" >> /home/kubernetes/k8s_auth_data/known_tokens.csv\" && \ + sudo bash -c \"echo ${KUBE_PASSWORD},admin,admin > /home/kubernetes/k8s_auth_data/basic_auth.csv\"" execute-cmd-on-master-with-retries "${PKI_SETUP_CMD}" 3 echo "Wrote PKI certs, keys, tokens and admin password to master." }