Fix "validate-cluster.sh" to use "kubectl get componentstatus".

This commit is contained in:
Fabio Yeon 2015-04-15 13:04:22 -07:00
parent 951a125751
commit c6ef2ddf8a
32 changed files with 65 additions and 96 deletions

View File

@ -14,11 +14,7 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
# Bring up a Kubernetes cluster. # Validates that the cluster is healthy.
#
# If the full release name (gs://<bucket>/<release>) is passed in then we take
# that directly. If not then we assume we are doing development stuff and take
# the defaults in the release config.
set -o errexit set -o errexit
set -o nounset set -o nounset
@ -28,12 +24,9 @@ KUBE_ROOT=$(dirname "${BASH_SOURCE}")/..
source "${KUBE_ROOT}/cluster/kube-env.sh" source "${KUBE_ROOT}/cluster/kube-env.sh"
source "${KUBE_ROOT}/cluster/${KUBERNETES_PROVIDER}/util.sh" source "${KUBE_ROOT}/cluster/${KUBERNETES_PROVIDER}/util.sh"
get-password
detect-master > /dev/null
detect-minions > /dev/null
MINIONS_FILE=/tmp/minions-$$ MINIONS_FILE=/tmp/minions-$$
trap 'rm -rf "${MINIONS_FILE}"' EXIT trap 'rm -rf "${MINIONS_FILE}"' EXIT
# Make several attempts to deal with slow cluster birth. # Make several attempts to deal with slow cluster birth.
attempt=0 attempt=0
while true; do while true; do
@ -54,62 +47,39 @@ done
echo "Found ${found} nodes." echo "Found ${found} nodes."
cat -n "${MINIONS_FILE}" cat -n "${MINIONS_FILE}"
# On vSphere, use minion IPs as their names attempt=0
if [[ "${KUBERNETES_PROVIDER}" == "vsphere" || "${KUBERNETES_PROVIDER}" == "vagrant" || "${KUBERNETES_PROVIDER}" == "libvirt-coreos" || "${KUBERNETES_PROVIDER}" == "juju" ]] ; then while true; do
MINION_NAMES=("${KUBE_MINION_IP_ADDRESSES[@]}") kubectl_output=$("${KUBE_ROOT}/cluster/kubectl.sh" get componentstatus)
fi
# On AWS we can't really name the minions, so just trust that if the number is right, the right names are there. # The "kubectl validate" output is four columns like this:
if [[ "${KUBERNETES_PROVIDER}" == "aws" ]]; then #
MINION_NAMES=("$(cat ${MINIONS_FILE})") # COMPONENT HEALTH MSG ERR
# /healthz validation isn't working for some reason on AWS. So just hope for the best. # controller-manager Healthy ok nil
# TODO: figure out why and fix, it must be working in some form, or else clusters wouldn't work. #
echo "Kubelet health checking on AWS isn't currently supported, assuming everything is good..." # Parse the output to capture the value of the second column("HEALTH"), then use grep to
echo -e "${color_green}Cluster validation succeeded${color_norm}" # count the number of times it doesn't match "success".
exit 0 # Because of the header, the actual unsuccessful count is 1 minus the count.
fi
for (( i=0; i<${#MINION_NAMES[@]}; i++)); do non_success_count=$(echo "${kubectl_output}" | \
# Grep returns an exit status of 1 when line is not found, so we need the : to always return a 0 exit status sed -n 's/^\([[:alnum:][:punct:]]\+\)\s\+\([[:alnum:][:punct:]]\+\)\s\+.*/\2/p' | \
count=$(grep -c "${MINION_NAMES[$i]}" "${MINIONS_FILE}") || : grep 'Healthy' --invert-match -c)
if [[ "${count}" == "0" ]]; then
echo -e "${color_red}Failed to find ${MINION_NAMES[$i]}, cluster is probably broken.${color_norm}"
cat -n "${MINIONS_FILE}"
exit 1
fi
name="${MINION_NAMES[$i]}" if ((non_success_count > 1)); then
if [[ "$KUBERNETES_PROVIDER" != "vsphere" && "$KUBERNETES_PROVIDER" != "vagrant" && "$KUBERNETES_PROVIDER" != "libvirt-coreos" && "$KUBERNETES_PROVIDER" != "juju" ]]; then if ((attempt < 5)); then
# Grab fully qualified name echo -e "${color_yellow}Cluster not working yet.${color_norm}"
name=$(grep "${MINION_NAMES[$i]}\." "${MINIONS_FILE}")
fi
# Make sure the kubelet is healthy.
# Make several attempts to deal with slow cluster birth.
attempt=0
while true; do
echo -n "Attempt $((attempt+1)) at checking Kubelet installation on node ${MINION_NAMES[$i]} ..."
if [[ "$KUBERNETES_PROVIDER" != "libvirt-coreos" && "$KUBERNETES_PROVIDER" != "juju" ]]; then
curl_output=$(curl -s --insecure --user "${KUBE_USER}:${KUBE_PASSWORD}" \
"https://${KUBE_MASTER_IP}/api/v1beta1/proxy/minions/${name}/healthz")
else
curl_output=$(curl -s \
"http://${KUBE_MASTER_IP}:8080/api/v1beta1/proxy/minions/${name}/healthz")
fi
if [[ "${curl_output}" != "ok" ]]; then
if (( attempt > 5 )); then
echo
echo -e "${color_red}Kubelet failed to install on node ${MINION_NAMES[$i]}. Your cluster is unlikely to work correctly."
echo -e "Please run ./cluster/kube-down.sh and re-create the cluster. (sorry!)${color_norm}"
exit 1
fi
else
echo -e " ${color_green}[working]${color_norm}"
break
fi
echo -e " ${color_yellow}[not working yet]${color_norm}"
attempt=$((attempt+1)) attempt=$((attempt+1))
sleep 30 sleep 30
done else
echo -e " ${color_yellow}Validate output:${color_norm}"
echo "${kubectl_output}"
echo -e "${color_red}Validation returned one or more failed components. Cluster is probably broken.${color_norm}"
exit 1
fi
else
break
fi
done done
echo "Validate output:"
echo "${kubectl_output}"
echo -e "${color_green}Cluster validation succeeded${color_norm}" echo -e "${color_green}Cluster validation succeeded${color_norm}"

View File

@ -66,4 +66,4 @@ kubectl
* [kubectl update](kubectl_update.md) - Update a resource by filename or stdin. * [kubectl update](kubectl_update.md) - Update a resource by filename or stdin.
* [kubectl version](kubectl_version.md) - Print the client and server version information. * [kubectl version](kubectl_version.md) - Print the client and server version information.
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.392549632 +0000 UTC ###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.488963312 +0000 UTC

View File

@ -50,4 +50,4 @@ kubectl api-versions
### SEE ALSO ### SEE ALSO
* [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager * [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.39227534 +0000 UTC ###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.488505223 +0000 UTC

View File

@ -50,4 +50,4 @@ kubectl cluster-info
### SEE ALSO ### SEE ALSO
* [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager * [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.392162759 +0000 UTC ###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.48831375 +0000 UTC

View File

@ -63,4 +63,4 @@ kubectl config SUBCOMMAND
* [kubectl config use-context](kubectl_config_use-context.md) - Sets the current-context in a kubeconfig file * [kubectl config use-context](kubectl_config_use-context.md) - Sets the current-context in a kubeconfig file
* [kubectl config view](kubectl_config_view.md) - displays Merged kubeconfig settings or a specified kubeconfig file. * [kubectl config view](kubectl_config_view.md) - displays Merged kubeconfig settings or a specified kubeconfig file.
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.392043616 +0000 UTC ###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.488116168 +0000 UTC

View File

@ -65,4 +65,4 @@ $ kubectl config set-cluster e2e --insecure-skip-tls-verify=true
### SEE ALSO ### SEE ALSO
* [kubectl config](kubectl_config.md) - config modifies kubeconfig files * [kubectl config](kubectl_config.md) - config modifies kubeconfig files
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.39119629 +0000 UTC ###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.486460859 +0000 UTC

View File

@ -58,4 +58,4 @@ $ kubectl config set-context gce --user=cluster-admin
### SEE ALSO ### SEE ALSO
* [kubectl config](kubectl_config.md) - config modifies kubeconfig files * [kubectl config](kubectl_config.md) - config modifies kubeconfig files
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.391488399 +0000 UTC ###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.486736724 +0000 UTC

View File

@ -78,4 +78,4 @@ $ kubectl set-credentials cluster-admin --client-certificate=~/.kube/admin.crt -
### SEE ALSO ### SEE ALSO
* [kubectl config](kubectl_config.md) - config modifies kubeconfig files * [kubectl config](kubectl_config.md) - config modifies kubeconfig files
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.391323192 +0000 UTC ###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.486604006 +0000 UTC

View File

@ -52,4 +52,4 @@ kubectl config set PROPERTY_NAME PROPERTY_VALUE
### SEE ALSO ### SEE ALSO
* [kubectl config](kubectl_config.md) - config modifies kubeconfig files * [kubectl config](kubectl_config.md) - config modifies kubeconfig files
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.391618859 +0000 UTC ###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.486861123 +0000 UTC

View File

@ -51,4 +51,4 @@ kubectl config unset PROPERTY_NAME
### SEE ALSO ### SEE ALSO
* [kubectl config](kubectl_config.md) - config modifies kubeconfig files * [kubectl config](kubectl_config.md) - config modifies kubeconfig files
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.391735806 +0000 UTC ###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.487685494 +0000 UTC

View File

@ -50,4 +50,4 @@ kubectl config use-context CONTEXT_NAME
### SEE ALSO ### SEE ALSO
* [kubectl config](kubectl_config.md) - config modifies kubeconfig files * [kubectl config](kubectl_config.md) - config modifies kubeconfig files
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.391848246 +0000 UTC ###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.487888021 +0000 UTC

View File

@ -72,4 +72,4 @@ $ kubectl config view -o template --template='{{range .users}}{{ if eq .name "e2
### SEE ALSO ### SEE ALSO
* [kubectl config](kubectl_config.md) - config modifies kubeconfig files * [kubectl config](kubectl_config.md) - config modifies kubeconfig files
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.391073075 +0000 UTC ###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.486319118 +0000 UTC

View File

@ -63,4 +63,4 @@ $ cat pod.json | kubectl create -f -
### SEE ALSO ### SEE ALSO
* [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager * [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.388588064 +0000 UTC ###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.48343431 +0000 UTC

View File

@ -81,4 +81,4 @@ $ kubectl delete pods --all
### SEE ALSO ### SEE ALSO
* [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager * [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.389412973 +0000 UTC ###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.483731878 +0000 UTC

View File

@ -53,4 +53,4 @@ kubectl describe RESOURCE ID
### SEE ALSO ### SEE ALSO
* [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager * [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.388410556 +0000 UTC ###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.483293174 +0000 UTC

View File

@ -64,4 +64,4 @@ $ kubectl exec -p 123456-7890 -c ruby-container -i -t -- bash -il
### SEE ALSO ### SEE ALSO
* [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager * [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.390127525 +0000 UTC ###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.484697863 +0000 UTC

View File

@ -82,4 +82,4 @@ $ kubectl expose streamer --port=4100 --protocol=udp --service-name=video-stream
### SEE ALSO ### SEE ALSO
* [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager * [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.390792874 +0000 UTC ###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.485803902 +0000 UTC

View File

@ -8,7 +8,7 @@ Display one or many resources
Display one or many resources. Display one or many resources.
Possible resources include pods (po), replication controllers (rc), services Possible resources include pods (po), replication controllers (rc), services
(svc), minions (mi), or events (ev). (svc), minions (mi), events (ev), or component statuses (cs).
By specifying the output as 'template' and providing a Go template as the value By specifying the output as 'template' and providing a Go template as the value
of the --template flag, you can filter the attributes of the fetched resource(s). of the --template flag, you can filter the attributes of the fetched resource(s).
@ -85,4 +85,4 @@ $ kubectl get rc/web service/frontend pods/web-pod-13je7
### SEE ALSO ### SEE ALSO
* [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager * [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.387483074 +0000 UTC ###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.482589064 +0000 UTC

View File

@ -81,4 +81,4 @@ $ kubectl label pods foo bar-
### SEE ALSO ### SEE ALSO
* [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager * [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.390937166 +0000 UTC ###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.486060232 +0000 UTC

View File

@ -62,4 +62,4 @@ $ kubectl log -f 123456-7890 ruby-container
### SEE ALSO ### SEE ALSO
* [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager * [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.389728881 +0000 UTC ###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.484139739 +0000 UTC

View File

@ -53,4 +53,4 @@ kubectl namespace [namespace]
### SEE ALSO ### SEE ALSO
* [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager * [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.389609191 +0000 UTC ###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.483937463 +0000 UTC

View File

@ -68,4 +68,4 @@ $ kubectl port-forward -p mypod 0:5000
### SEE ALSO ### SEE ALSO
* [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager * [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.390241417 +0000 UTC ###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.484899751 +0000 UTC

View File

@ -65,4 +65,4 @@ $ kubectl proxy --api-prefix=k8s-api
### SEE ALSO ### SEE ALSO
* [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager * [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.390360738 +0000 UTC ###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.485099157 +0000 UTC

View File

@ -68,4 +68,4 @@ $ kubectl resize --current-replicas=2 --replicas=3 replicationcontrollers foo
### SEE ALSO ### SEE ALSO
* [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager * [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.389989377 +0000 UTC ###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.484493463 +0000 UTC

View File

@ -68,4 +68,4 @@ $ cat frontend-v2.json | kubectl rolling-update frontend-v1 -f -
### SEE ALSO ### SEE ALSO
* [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager * [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.38985117 +0000 UTC ###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.484316119 +0000 UTC

View File

@ -78,4 +78,4 @@ $ kubectl run-container nginx --image=nginx --overrides='{ "apiVersion": "v1beta
### SEE ALSO ### SEE ALSO
* [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager * [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.390501802 +0000 UTC ###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.485362986 +0000 UTC

View File

@ -72,4 +72,4 @@ $ kubectl stop -f path/to/resources
### SEE ALSO ### SEE ALSO
* [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager * [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.390631789 +0000 UTC ###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.48555328 +0000 UTC

View File

@ -67,4 +67,4 @@ $ kubectl update pods my-pod --patch='{ "apiVersion": "v1beta1", "desiredState":
### SEE ALSO ### SEE ALSO
* [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager * [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.388743178 +0000 UTC ###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.483572524 +0000 UTC

View File

@ -51,4 +51,4 @@ kubectl version
### SEE ALSO ### SEE ALSO
* [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager * [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.392395408 +0000 UTC ###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.488692518 +0000 UTC

View File

@ -17,7 +17,7 @@ Display one or many resources.
.PP .PP
Possible resources include pods (po), replication controllers (rc), services Possible resources include pods (po), replication controllers (rc), services
(svc), minions (mi), or events (ev). (svc), minions (mi), events (ev), or component statuses (cs).
.PP .PP
By specifying the output as 'template' and providing a Go template as the value By specifying the output as 'template' and providing a Go template as the value

View File

@ -1682,8 +1682,8 @@ const (
) )
type ComponentCondition struct { type ComponentCondition struct {
Type ComponentConditionType `json:"type"` Type ComponentConditionType `json:"type" description:"type of component condition, currently only Healthy"`
Status ConditionStatus `json:"status"` Status ConditionStatus `json:"status" description:"current status of this component condition"`
Message string `json:"message,omitempty" description:"health check message received from the component"` Message string `json:"message,omitempty" description:"health check message received from the component"`
Error string `json:"error,omitempty" description:"error code from health check attempt (if any)"` Error string `json:"error,omitempty" description:"error code from health check attempt (if any)"`
} }
@ -1694,12 +1694,12 @@ type ComponentStatus struct {
ObjectMeta `json:"metadata,omitempty"` ObjectMeta `json:"metadata,omitempty"`
Name string `json:"name,omitempty" description:"name of the component"` Name string `json:"name,omitempty" description:"name of the component"`
Conditions []ComponentCondition `json:"conditions,omitempty"` Conditions []ComponentCondition `json:"conditions,omitempty" description:"Conditions is a list of ComponentCondition objects"`
} }
type ComponentStatusList struct { type ComponentStatusList struct {
TypeMeta `json:",inline"` TypeMeta `json:",inline"`
ListMeta `json:"metadata,omitempty"` ListMeta `json:"metadata,omitempty"`
Items []ComponentStatus `json:"items" description:"items is a list of component status objects"` Items []ComponentStatus `json:"items" description:"items is a list of ComponentStatus objects"`
} }

View File

@ -65,7 +65,6 @@ func TestValidate(t *testing.T) {
StatusCode: test.code, StatusCode: test.code,
}, },
} }
// fake := &http.Client{Transport: fakeRT}
status, data, err := s.DoServerCheck(fakeRT) status, data, err := s.DoServerCheck(fakeRT)
expect := fmt.Sprintf("http://%s:%d/healthz", s.Addr, s.Port) expect := fmt.Sprintf("http://%s:%d/healthz", s.Addr, s.Port)
if fakeRT.url != expect { if fakeRT.url != expect {