Fix "validate-cluster.sh" to use "kubectl get componentstatus".

This commit is contained in:
Fabio Yeon 2015-04-15 13:04:22 -07:00
parent 951a125751
commit c6ef2ddf8a
32 changed files with 65 additions and 96 deletions

View File

@ -14,11 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# Bring up a Kubernetes cluster.
#
# If the full release name (gs://<bucket>/<release>) is passed in then we take
# that directly. If not then we assume we are doing development stuff and take
# the defaults in the release config.
# Validates that the cluster is healthy.
set -o errexit
set -o nounset
@ -28,12 +24,9 @@ KUBE_ROOT=$(dirname "${BASH_SOURCE}")/..
source "${KUBE_ROOT}/cluster/kube-env.sh"
source "${KUBE_ROOT}/cluster/${KUBERNETES_PROVIDER}/util.sh"
get-password
detect-master > /dev/null
detect-minions > /dev/null
MINIONS_FILE=/tmp/minions-$$
trap 'rm -rf "${MINIONS_FILE}"' EXIT
# Make several attempts to deal with slow cluster birth.
attempt=0
while true; do
@ -54,62 +47,39 @@ done
echo "Found ${found} nodes."
cat -n "${MINIONS_FILE}"
# On vSphere, use minion IPs as their names
if [[ "${KUBERNETES_PROVIDER}" == "vsphere" || "${KUBERNETES_PROVIDER}" == "vagrant" || "${KUBERNETES_PROVIDER}" == "libvirt-coreos" || "${KUBERNETES_PROVIDER}" == "juju" ]] ; then
MINION_NAMES=("${KUBE_MINION_IP_ADDRESSES[@]}")
fi
attempt=0
while true; do
kubectl_output=$("${KUBE_ROOT}/cluster/kubectl.sh" get componentstatus)
# On AWS we can't really name the minions, so just trust that if the number is right, the right names are there.
if [[ "${KUBERNETES_PROVIDER}" == "aws" ]]; then
MINION_NAMES=("$(cat ${MINIONS_FILE})")
# /healthz validation isn't working for some reason on AWS. So just hope for the best.
# TODO: figure out why and fix, it must be working in some form, or else clusters wouldn't work.
echo "Kubelet health checking on AWS isn't currently supported, assuming everything is good..."
echo -e "${color_green}Cluster validation succeeded${color_norm}"
exit 0
fi
# The "kubectl validate" output is four columns like this:
#
# COMPONENT HEALTH MSG ERR
# controller-manager Healthy ok nil
#
# Parse the output to capture the value of the second column("HEALTH"), then use grep to
# count the number of times it doesn't match "success".
# Because of the header, the actual unsuccessful count is 1 minus the count.
for (( i=0; i<${#MINION_NAMES[@]}; i++)); do
# Grep returns an exit status of 1 when line is not found, so we need the : to always return a 0 exit status
count=$(grep -c "${MINION_NAMES[$i]}" "${MINIONS_FILE}") || :
if [[ "${count}" == "0" ]]; then
echo -e "${color_red}Failed to find ${MINION_NAMES[$i]}, cluster is probably broken.${color_norm}"
cat -n "${MINIONS_FILE}"
exit 1
fi
non_success_count=$(echo "${kubectl_output}" | \
sed -n 's/^\([[:alnum:][:punct:]]\+\)\s\+\([[:alnum:][:punct:]]\+\)\s\+.*/\2/p' | \
grep 'Healthy' --invert-match -c)
name="${MINION_NAMES[$i]}"
if [[ "$KUBERNETES_PROVIDER" != "vsphere" && "$KUBERNETES_PROVIDER" != "vagrant" && "$KUBERNETES_PROVIDER" != "libvirt-coreos" && "$KUBERNETES_PROVIDER" != "juju" ]]; then
# Grab fully qualified name
name=$(grep "${MINION_NAMES[$i]}\." "${MINIONS_FILE}")
fi
# Make sure the kubelet is healthy.
# Make several attempts to deal with slow cluster birth.
attempt=0
while true; do
echo -n "Attempt $((attempt+1)) at checking Kubelet installation on node ${MINION_NAMES[$i]} ..."
if [[ "$KUBERNETES_PROVIDER" != "libvirt-coreos" && "$KUBERNETES_PROVIDER" != "juju" ]]; then
curl_output=$(curl -s --insecure --user "${KUBE_USER}:${KUBE_PASSWORD}" \
"https://${KUBE_MASTER_IP}/api/v1beta1/proxy/minions/${name}/healthz")
else
curl_output=$(curl -s \
"http://${KUBE_MASTER_IP}:8080/api/v1beta1/proxy/minions/${name}/healthz")
fi
if [[ "${curl_output}" != "ok" ]]; then
if (( attempt > 5 )); then
echo
echo -e "${color_red}Kubelet failed to install on node ${MINION_NAMES[$i]}. Your cluster is unlikely to work correctly."
echo -e "Please run ./cluster/kube-down.sh and re-create the cluster. (sorry!)${color_norm}"
exit 1
fi
else
echo -e " ${color_green}[working]${color_norm}"
break
fi
echo -e " ${color_yellow}[not working yet]${color_norm}"
if ((non_success_count > 1)); then
if ((attempt < 5)); then
echo -e "${color_yellow}Cluster not working yet.${color_norm}"
attempt=$((attempt+1))
sleep 30
done
else
echo -e " ${color_yellow}Validate output:${color_norm}"
echo "${kubectl_output}"
echo -e "${color_red}Validation returned one or more failed components. Cluster is probably broken.${color_norm}"
exit 1
fi
else
break
fi
done
echo "Validate output:"
echo "${kubectl_output}"
echo -e "${color_green}Cluster validation succeeded${color_norm}"

View File

@ -66,4 +66,4 @@ kubectl
* [kubectl update](kubectl_update.md) - Update a resource by filename or stdin.
* [kubectl version](kubectl_version.md) - Print the client and server version information.
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.392549632 +0000 UTC
###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.488963312 +0000 UTC

View File

@ -50,4 +50,4 @@ kubectl api-versions
### SEE ALSO
* [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.39227534 +0000 UTC
###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.488505223 +0000 UTC

View File

@ -50,4 +50,4 @@ kubectl cluster-info
### SEE ALSO
* [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.392162759 +0000 UTC
###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.48831375 +0000 UTC

View File

@ -63,4 +63,4 @@ kubectl config SUBCOMMAND
* [kubectl config use-context](kubectl_config_use-context.md) - Sets the current-context in a kubeconfig file
* [kubectl config view](kubectl_config_view.md) - displays Merged kubeconfig settings or a specified kubeconfig file.
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.392043616 +0000 UTC
###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.488116168 +0000 UTC

View File

@ -65,4 +65,4 @@ $ kubectl config set-cluster e2e --insecure-skip-tls-verify=true
### SEE ALSO
* [kubectl config](kubectl_config.md) - config modifies kubeconfig files
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.39119629 +0000 UTC
###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.486460859 +0000 UTC

View File

@ -58,4 +58,4 @@ $ kubectl config set-context gce --user=cluster-admin
### SEE ALSO
* [kubectl config](kubectl_config.md) - config modifies kubeconfig files
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.391488399 +0000 UTC
###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.486736724 +0000 UTC

View File

@ -78,4 +78,4 @@ $ kubectl set-credentials cluster-admin --client-certificate=~/.kube/admin.crt -
### SEE ALSO
* [kubectl config](kubectl_config.md) - config modifies kubeconfig files
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.391323192 +0000 UTC
###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.486604006 +0000 UTC

View File

@ -52,4 +52,4 @@ kubectl config set PROPERTY_NAME PROPERTY_VALUE
### SEE ALSO
* [kubectl config](kubectl_config.md) - config modifies kubeconfig files
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.391618859 +0000 UTC
###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.486861123 +0000 UTC

View File

@ -51,4 +51,4 @@ kubectl config unset PROPERTY_NAME
### SEE ALSO
* [kubectl config](kubectl_config.md) - config modifies kubeconfig files
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.391735806 +0000 UTC
###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.487685494 +0000 UTC

View File

@ -50,4 +50,4 @@ kubectl config use-context CONTEXT_NAME
### SEE ALSO
* [kubectl config](kubectl_config.md) - config modifies kubeconfig files
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.391848246 +0000 UTC
###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.487888021 +0000 UTC

View File

@ -72,4 +72,4 @@ $ kubectl config view -o template --template='{{range .users}}{{ if eq .name "e2
### SEE ALSO
* [kubectl config](kubectl_config.md) - config modifies kubeconfig files
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.391073075 +0000 UTC
###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.486319118 +0000 UTC

View File

@ -63,4 +63,4 @@ $ cat pod.json | kubectl create -f -
### SEE ALSO
* [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.388588064 +0000 UTC
###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.48343431 +0000 UTC

View File

@ -81,4 +81,4 @@ $ kubectl delete pods --all
### SEE ALSO
* [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.389412973 +0000 UTC
###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.483731878 +0000 UTC

View File

@ -53,4 +53,4 @@ kubectl describe RESOURCE ID
### SEE ALSO
* [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.388410556 +0000 UTC
###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.483293174 +0000 UTC

View File

@ -64,4 +64,4 @@ $ kubectl exec -p 123456-7890 -c ruby-container -i -t -- bash -il
### SEE ALSO
* [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.390127525 +0000 UTC
###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.484697863 +0000 UTC

View File

@ -82,4 +82,4 @@ $ kubectl expose streamer --port=4100 --protocol=udp --service-name=video-stream
### SEE ALSO
* [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.390792874 +0000 UTC
###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.485803902 +0000 UTC

View File

@ -8,7 +8,7 @@ Display one or many resources
Display one or many resources.
Possible resources include pods (po), replication controllers (rc), services
(svc), minions (mi), or events (ev).
(svc), minions (mi), events (ev), or component statuses (cs).
By specifying the output as 'template' and providing a Go template as the value
of the --template flag, you can filter the attributes of the fetched resource(s).
@ -85,4 +85,4 @@ $ kubectl get rc/web service/frontend pods/web-pod-13je7
### SEE ALSO
* [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.387483074 +0000 UTC
###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.482589064 +0000 UTC

View File

@ -81,4 +81,4 @@ $ kubectl label pods foo bar-
### SEE ALSO
* [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.390937166 +0000 UTC
###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.486060232 +0000 UTC

View File

@ -62,4 +62,4 @@ $ kubectl log -f 123456-7890 ruby-container
### SEE ALSO
* [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.389728881 +0000 UTC
###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.484139739 +0000 UTC

View File

@ -53,4 +53,4 @@ kubectl namespace [namespace]
### SEE ALSO
* [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.389609191 +0000 UTC
###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.483937463 +0000 UTC

View File

@ -68,4 +68,4 @@ $ kubectl port-forward -p mypod 0:5000
### SEE ALSO
* [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.390241417 +0000 UTC
###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.484899751 +0000 UTC

View File

@ -65,4 +65,4 @@ $ kubectl proxy --api-prefix=k8s-api
### SEE ALSO
* [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.390360738 +0000 UTC
###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.485099157 +0000 UTC

View File

@ -68,4 +68,4 @@ $ kubectl resize --current-replicas=2 --replicas=3 replicationcontrollers foo
### SEE ALSO
* [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.389989377 +0000 UTC
###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.484493463 +0000 UTC

View File

@ -68,4 +68,4 @@ $ cat frontend-v2.json | kubectl rolling-update frontend-v1 -f -
### SEE ALSO
* [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.38985117 +0000 UTC
###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.484316119 +0000 UTC

View File

@ -78,4 +78,4 @@ $ kubectl run-container nginx --image=nginx --overrides='{ "apiVersion": "v1beta
### SEE ALSO
* [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.390501802 +0000 UTC
###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.485362986 +0000 UTC

View File

@ -72,4 +72,4 @@ $ kubectl stop -f path/to/resources
### SEE ALSO
* [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.390631789 +0000 UTC
###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.48555328 +0000 UTC

View File

@ -67,4 +67,4 @@ $ kubectl update pods my-pod --patch='{ "apiVersion": "v1beta1", "desiredState":
### SEE ALSO
* [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.388743178 +0000 UTC
###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.483572524 +0000 UTC

View File

@ -51,4 +51,4 @@ kubectl version
### SEE ALSO
* [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.392395408 +0000 UTC
###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.488692518 +0000 UTC

View File

@ -17,7 +17,7 @@ Display one or many resources.
.PP
Possible resources include pods (po), replication controllers (rc), services
(svc), minions (mi), or events (ev).
(svc), minions (mi), events (ev), or component statuses (cs).
.PP
By specifying the output as 'template' and providing a Go template as the value

View File

@ -1682,8 +1682,8 @@ const (
)
type ComponentCondition struct {
Type ComponentConditionType `json:"type"`
Status ConditionStatus `json:"status"`
Type ComponentConditionType `json:"type" description:"type of component condition, currently only Healthy"`
Status ConditionStatus `json:"status" description:"current status of this component condition"`
Message string `json:"message,omitempty" description:"health check message received from the component"`
Error string `json:"error,omitempty" description:"error code from health check attempt (if any)"`
}
@ -1694,12 +1694,12 @@ type ComponentStatus struct {
ObjectMeta `json:"metadata,omitempty"`
Name string `json:"name,omitempty" description:"name of the component"`
Conditions []ComponentCondition `json:"conditions,omitempty"`
Conditions []ComponentCondition `json:"conditions,omitempty" description:"Conditions is a list of ComponentCondition objects"`
}
type ComponentStatusList struct {
TypeMeta `json:",inline"`
ListMeta `json:"metadata,omitempty"`
Items []ComponentStatus `json:"items" description:"items is a list of component status objects"`
Items []ComponentStatus `json:"items" description:"items is a list of ComponentStatus objects"`
}

View File

@ -65,7 +65,6 @@ func TestValidate(t *testing.T) {
StatusCode: test.code,
},
}
// fake := &http.Client{Transport: fakeRT}
status, data, err := s.DoServerCheck(fakeRT)
expect := fmt.Sprintf("http://%s:%d/healthz", s.Addr, s.Port)
if fakeRT.url != expect {