mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-24 12:15:52 +00:00
Fix "validate-cluster.sh" to use "kubectl get componentstatus".
This commit is contained in:
Fabio Yeon
parent
951a125751
commit
c6ef2ddf8a
@ -14,11 +14,7 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Bring up a Kubernetes cluster.
|
||||
#
|
||||
# If the full release name (gs://<bucket>/<release>) is passed in then we take
|
||||
# that directly. If not then we assume we are doing development stuff and take
|
||||
# the defaults in the release config.
|
||||
# Validates that the cluster is healthy.
|
||||
|
||||
set -o errexit
|
||||
set -o nounset
|
||||
@ -28,12 +24,9 @@ KUBE_ROOT=$(dirname "${BASH_SOURCE}")/..
|
||||
source "${KUBE_ROOT}/cluster/kube-env.sh"
|
||||
source "${KUBE_ROOT}/cluster/${KUBERNETES_PROVIDER}/util.sh"
|
||||
|
||||
get-password
|
||||
detect-master > /dev/null
|
||||
detect-minions > /dev/null
|
||||
|
||||
MINIONS_FILE=/tmp/minions-$$
|
||||
trap 'rm -rf "${MINIONS_FILE}"' EXIT
|
||||
|
||||
# Make several attempts to deal with slow cluster birth.
|
||||
attempt=0
|
||||
while true; do
|
||||
@ -54,62 +47,39 @@ done
|
||||
echo "Found ${found} nodes."
|
||||
cat -n "${MINIONS_FILE}"
|
||||
|
||||
# On vSphere, use minion IPs as their names
|
||||
if [[ "${KUBERNETES_PROVIDER}" == "vsphere" || "${KUBERNETES_PROVIDER}" == "vagrant" || "${KUBERNETES_PROVIDER}" == "libvirt-coreos" || "${KUBERNETES_PROVIDER}" == "juju" ]] ; then
|
||||
MINION_NAMES=("${KUBE_MINION_IP_ADDRESSES[@]}")
|
||||
fi
|
||||
attempt=0
|
||||
while true; do
|
||||
kubectl_output=$("${KUBE_ROOT}/cluster/kubectl.sh" get componentstatus)
|
||||
|
||||
# On AWS we can't really name the minions, so just trust that if the number is right, the right names are there.
|
||||
if [[ "${KUBERNETES_PROVIDER}" == "aws" ]]; then
|
||||
MINION_NAMES=("$(cat ${MINIONS_FILE})")
|
||||
# /healthz validation isn't working for some reason on AWS. So just hope for the best.
|
||||
# TODO: figure out why and fix, it must be working in some form, or else clusters wouldn't work.
|
||||
echo "Kubelet health checking on AWS isn't currently supported, assuming everything is good..."
|
||||
echo -e "${color_green}Cluster validation succeeded${color_norm}"
|
||||
exit 0
|
||||
fi
|
||||
# The "kubectl validate" output is four columns like this:
|
||||
#
|
||||
# COMPONENT HEALTH MSG ERR
|
||||
# controller-manager Healthy ok nil
|
||||
#
|
||||
# Parse the output to capture the value of the second column("HEALTH"), then use grep to
|
||||
# count the number of times it doesn't match "success".
|
||||
# Because of the header, the actual unsuccessful count is 1 minus the count.
|
||||
|
||||
for (( i=0; i<${#MINION_NAMES[@]}; i++)); do
|
||||
# Grep returns an exit status of 1 when line is not found, so we need the : to always return a 0 exit status
|
||||
count=$(grep -c "${MINION_NAMES[$i]}" "${MINIONS_FILE}") || :
|
||||
if [[ "${count}" == "0" ]]; then
|
||||
echo -e "${color_red}Failed to find ${MINION_NAMES[$i]}, cluster is probably broken.${color_norm}"
|
||||
cat -n "${MINIONS_FILE}"
|
||||
exit 1
|
||||
fi
|
||||
non_success_count=$(echo "${kubectl_output}" | \
|
||||
sed -n 's/^\([[:alnum:][:punct:]]\+\)\s\+\([[:alnum:][:punct:]]\+\)\s\+.*/\2/p' | \
|
||||
grep 'Healthy' --invert-match -c)
|
||||
|
||||
name="${MINION_NAMES[$i]}"
|
||||
if [[ "$KUBERNETES_PROVIDER" != "vsphere" && "$KUBERNETES_PROVIDER" != "vagrant" && "$KUBERNETES_PROVIDER" != "libvirt-coreos" && "$KUBERNETES_PROVIDER" != "juju" ]]; then
|
||||
# Grab fully qualified name
|
||||
name=$(grep "${MINION_NAMES[$i]}\." "${MINIONS_FILE}")
|
||||
fi
|
||||
|
||||
# Make sure the kubelet is healthy.
|
||||
# Make several attempts to deal with slow cluster birth.
|
||||
attempt=0
|
||||
while true; do
|
||||
echo -n "Attempt $((attempt+1)) at checking Kubelet installation on node ${MINION_NAMES[$i]} ..."
|
||||
if [[ "$KUBERNETES_PROVIDER" != "libvirt-coreos" && "$KUBERNETES_PROVIDER" != "juju" ]]; then
|
||||
curl_output=$(curl -s --insecure --user "${KUBE_USER}:${KUBE_PASSWORD}" \
|
||||
"https://${KUBE_MASTER_IP}/api/v1beta1/proxy/minions/${name}/healthz")
|
||||
else
|
||||
curl_output=$(curl -s \
|
||||
"http://${KUBE_MASTER_IP}:8080/api/v1beta1/proxy/minions/${name}/healthz")
|
||||
fi
|
||||
if [[ "${curl_output}" != "ok" ]]; then
|
||||
if (( attempt > 5 )); then
|
||||
echo
|
||||
echo -e "${color_red}Kubelet failed to install on node ${MINION_NAMES[$i]}. Your cluster is unlikely to work correctly."
|
||||
echo -e "Please run ./cluster/kube-down.sh and re-create the cluster. (sorry!)${color_norm}"
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
echo -e " ${color_green}[working]${color_norm}"
|
||||
break
|
||||
fi
|
||||
echo -e " ${color_yellow}[not working yet]${color_norm}"
|
||||
if ((non_success_count > 1)); then
|
||||
if ((attempt < 5)); then
|
||||
echo -e "${color_yellow}Cluster not working yet.${color_norm}"
|
||||
attempt=$((attempt+1))
|
||||
sleep 30
|
||||
done
|
||||
else
|
||||
echo -e " ${color_yellow}Validate output:${color_norm}"
|
||||
echo "${kubectl_output}"
|
||||
echo -e "${color_red}Validation returned one or more failed components. Cluster is probably broken.${color_norm}"
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
echo "Validate output:"
|
||||
echo "${kubectl_output}"
|
||||
echo -e "${color_green}Cluster validation succeeded${color_norm}"
|
||||
|
||||
@ -66,4 +66,4 @@ kubectl
|
||||
* [kubectl update](kubectl_update.md) - Update a resource by filename or stdin.
|
||||
* [kubectl version](kubectl_version.md) - Print the client and server version information.
|
||||
|
||||
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.392549632 +0000 UTC
|
||||
###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.488963312 +0000 UTC
|
||||
|
||||
@ -50,4 +50,4 @@ kubectl api-versions
|
||||
### SEE ALSO
|
||||
* [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager
|
||||
|
||||
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.39227534 +0000 UTC
|
||||
###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.488505223 +0000 UTC
|
||||
|
||||
@ -50,4 +50,4 @@ kubectl cluster-info
|
||||
### SEE ALSO
|
||||
* [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager
|
||||
|
||||
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.392162759 +0000 UTC
|
||||
###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.48831375 +0000 UTC
|
||||
|
||||
@ -63,4 +63,4 @@ kubectl config SUBCOMMAND
|
||||
* [kubectl config use-context](kubectl_config_use-context.md) - Sets the current-context in a kubeconfig file
|
||||
* [kubectl config view](kubectl_config_view.md) - displays Merged kubeconfig settings or a specified kubeconfig file.
|
||||
|
||||
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.392043616 +0000 UTC
|
||||
###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.488116168 +0000 UTC
|
||||
|
||||
@ -65,4 +65,4 @@ $ kubectl config set-cluster e2e --insecure-skip-tls-verify=true
|
||||
### SEE ALSO
|
||||
* [kubectl config](kubectl_config.md) - config modifies kubeconfig files
|
||||
|
||||
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.39119629 +0000 UTC
|
||||
###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.486460859 +0000 UTC
|
||||
|
||||
@ -58,4 +58,4 @@ $ kubectl config set-context gce --user=cluster-admin
|
||||
### SEE ALSO
|
||||
* [kubectl config](kubectl_config.md) - config modifies kubeconfig files
|
||||
|
||||
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.391488399 +0000 UTC
|
||||
###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.486736724 +0000 UTC
|
||||
|
||||
@ -78,4 +78,4 @@ $ kubectl set-credentials cluster-admin --client-certificate=~/.kube/admin.crt -
|
||||
### SEE ALSO
|
||||
* [kubectl config](kubectl_config.md) - config modifies kubeconfig files
|
||||
|
||||
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.391323192 +0000 UTC
|
||||
###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.486604006 +0000 UTC
|
||||
|
||||
@ -52,4 +52,4 @@ kubectl config set PROPERTY_NAME PROPERTY_VALUE
|
||||
### SEE ALSO
|
||||
* [kubectl config](kubectl_config.md) - config modifies kubeconfig files
|
||||
|
||||
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.391618859 +0000 UTC
|
||||
###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.486861123 +0000 UTC
|
||||
|
||||
@ -51,4 +51,4 @@ kubectl config unset PROPERTY_NAME
|
||||
### SEE ALSO
|
||||
* [kubectl config](kubectl_config.md) - config modifies kubeconfig files
|
||||
|
||||
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.391735806 +0000 UTC
|
||||
###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.487685494 +0000 UTC
|
||||
|
||||
@ -50,4 +50,4 @@ kubectl config use-context CONTEXT_NAME
|
||||
### SEE ALSO
|
||||
* [kubectl config](kubectl_config.md) - config modifies kubeconfig files
|
||||
|
||||
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.391848246 +0000 UTC
|
||||
###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.487888021 +0000 UTC
|
||||
|
||||
@ -72,4 +72,4 @@ $ kubectl config view -o template --template='{{range .users}}{{ if eq .name "e2
|
||||
### SEE ALSO
|
||||
* [kubectl config](kubectl_config.md) - config modifies kubeconfig files
|
||||
|
||||
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.391073075 +0000 UTC
|
||||
###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.486319118 +0000 UTC
|
||||
|
||||
@ -63,4 +63,4 @@ $ cat pod.json | kubectl create -f -
|
||||
### SEE ALSO
|
||||
* [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager
|
||||
|
||||
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.388588064 +0000 UTC
|
||||
###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.48343431 +0000 UTC
|
||||
|
||||
@ -81,4 +81,4 @@ $ kubectl delete pods --all
|
||||
### SEE ALSO
|
||||
* [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager
|
||||
|
||||
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.389412973 +0000 UTC
|
||||
###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.483731878 +0000 UTC
|
||||
|
||||
@ -53,4 +53,4 @@ kubectl describe RESOURCE ID
|
||||
### SEE ALSO
|
||||
* [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager
|
||||
|
||||
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.388410556 +0000 UTC
|
||||
###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.483293174 +0000 UTC
|
||||
|
||||
@ -64,4 +64,4 @@ $ kubectl exec -p 123456-7890 -c ruby-container -i -t -- bash -il
|
||||
### SEE ALSO
|
||||
* [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager
|
||||
|
||||
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.390127525 +0000 UTC
|
||||
###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.484697863 +0000 UTC
|
||||
|
||||
@ -82,4 +82,4 @@ $ kubectl expose streamer --port=4100 --protocol=udp --service-name=video-stream
|
||||
### SEE ALSO
|
||||
* [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager
|
||||
|
||||
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.390792874 +0000 UTC
|
||||
###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.485803902 +0000 UTC
|
||||
|
||||
@ -8,7 +8,7 @@ Display one or many resources
|
||||
Display one or many resources.
|
||||
|
||||
Possible resources include pods (po), replication controllers (rc), services
|
||||
(svc), minions (mi), or events (ev).
|
||||
(svc), minions (mi), events (ev), or component statuses (cs).
|
||||
|
||||
By specifying the output as 'template' and providing a Go template as the value
|
||||
of the --template flag, you can filter the attributes of the fetched resource(s).
|
||||
@ -85,4 +85,4 @@ $ kubectl get rc/web service/frontend pods/web-pod-13je7
|
||||
### SEE ALSO
|
||||
* [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager
|
||||
|
||||
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.387483074 +0000 UTC
|
||||
###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.482589064 +0000 UTC
|
||||
|
||||
@ -81,4 +81,4 @@ $ kubectl label pods foo bar-
|
||||
### SEE ALSO
|
||||
* [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager
|
||||
|
||||
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.390937166 +0000 UTC
|
||||
###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.486060232 +0000 UTC
|
||||
|
||||
@ -62,4 +62,4 @@ $ kubectl log -f 123456-7890 ruby-container
|
||||
### SEE ALSO
|
||||
* [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager
|
||||
|
||||
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.389728881 +0000 UTC
|
||||
###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.484139739 +0000 UTC
|
||||
|
||||
@ -53,4 +53,4 @@ kubectl namespace [namespace]
|
||||
### SEE ALSO
|
||||
* [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager
|
||||
|
||||
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.389609191 +0000 UTC
|
||||
###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.483937463 +0000 UTC
|
||||
|
||||
@ -68,4 +68,4 @@ $ kubectl port-forward -p mypod 0:5000
|
||||
### SEE ALSO
|
||||
* [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager
|
||||
|
||||
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.390241417 +0000 UTC
|
||||
###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.484899751 +0000 UTC
|
||||
|
||||
@ -65,4 +65,4 @@ $ kubectl proxy --api-prefix=k8s-api
|
||||
### SEE ALSO
|
||||
* [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager
|
||||
|
||||
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.390360738 +0000 UTC
|
||||
###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.485099157 +0000 UTC
|
||||
|
||||
@ -68,4 +68,4 @@ $ kubectl resize --current-replicas=2 --replicas=3 replicationcontrollers foo
|
||||
### SEE ALSO
|
||||
* [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager
|
||||
|
||||
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.389989377 +0000 UTC
|
||||
###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.484493463 +0000 UTC
|
||||
|
||||
@ -68,4 +68,4 @@ $ cat frontend-v2.json | kubectl rolling-update frontend-v1 -f -
|
||||
### SEE ALSO
|
||||
* [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager
|
||||
|
||||
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.38985117 +0000 UTC
|
||||
###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.484316119 +0000 UTC
|
||||
|
||||
@ -78,4 +78,4 @@ $ kubectl run-container nginx --image=nginx --overrides='{ "apiVersion": "v1beta
|
||||
### SEE ALSO
|
||||
* [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager
|
||||
|
||||
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.390501802 +0000 UTC
|
||||
###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.485362986 +0000 UTC
|
||||
|
||||
@ -72,4 +72,4 @@ $ kubectl stop -f path/to/resources
|
||||
### SEE ALSO
|
||||
* [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager
|
||||
|
||||
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.390631789 +0000 UTC
|
||||
###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.48555328 +0000 UTC
|
||||
|
||||
@ -67,4 +67,4 @@ $ kubectl update pods my-pod --patch='{ "apiVersion": "v1beta1", "desiredState":
|
||||
### SEE ALSO
|
||||
* [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager
|
||||
|
||||
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.388743178 +0000 UTC
|
||||
###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.483572524 +0000 UTC
|
||||
|
||||
@ -51,4 +51,4 @@ kubectl version
|
||||
### SEE ALSO
|
||||
* [kubectl](kubectl.md) - kubectl controls the Kubernetes cluster manager
|
||||
|
||||
###### Auto generated by spf13/cobra at 2015-04-16 17:04:37.392395408 +0000 UTC
|
||||
###### Auto generated by spf13/cobra at 2015-04-17 18:59:11.488692518 +0000 UTC
|
||||
|
||||
@ -17,7 +17,7 @@ Display one or many resources.
|
||||
|
||||
.PP
|
||||
Possible resources include pods (po), replication controllers (rc), services
|
||||
(svc), minions (mi), or events (ev).
|
||||
(svc), minions (mi), events (ev), or component statuses (cs).
|
||||
|
||||
.PP
|
||||
By specifying the output as 'template' and providing a Go template as the value
|
||||
|
||||
@ -1682,8 +1682,8 @@ const (
|
||||
)
|
||||
|
||||
type ComponentCondition struct {
|
||||
Type ComponentConditionType `json:"type"`
|
||||
Status ConditionStatus `json:"status"`
|
||||
Type ComponentConditionType `json:"type" description:"type of component condition, currently only Healthy"`
|
||||
Status ConditionStatus `json:"status" description:"current status of this component condition"`
|
||||
Message string `json:"message,omitempty" description:"health check message received from the component"`
|
||||
Error string `json:"error,omitempty" description:"error code from health check attempt (if any)"`
|
||||
}
|
||||
@ -1694,12 +1694,12 @@ type ComponentStatus struct {
|
||||
ObjectMeta `json:"metadata,omitempty"`
|
||||
|
||||
Name string `json:"name,omitempty" description:"name of the component"`
|
||||
Conditions []ComponentCondition `json:"conditions,omitempty"`
|
||||
Conditions []ComponentCondition `json:"conditions,omitempty" description:"Conditions is a list of ComponentCondition objects"`
|
||||
}
|
||||
|
||||
type ComponentStatusList struct {
|
||||
TypeMeta `json:",inline"`
|
||||
ListMeta `json:"metadata,omitempty"`
|
||||
|
||||
Items []ComponentStatus `json:"items" description:"items is a list of component status objects"`
|
||||
Items []ComponentStatus `json:"items" description:"items is a list of ComponentStatus objects"`
|
||||
}
|
||||
|
||||
@ -65,7 +65,6 @@ func TestValidate(t *testing.T) {
|
||||
StatusCode: test.code,
|
||||
},
|
||||
}
|
||||
// fake := &http.Client{Transport: fakeRT}
|
||||
status, data, err := s.DoServerCheck(fakeRT)
|
||||
expect := fmt.Sprintf("http://%s:%d/healthz", s.Addr, s.Port)
|
||||
if fakeRT.url != expect {
|
||||
|
||||
Loading…
Reference in New Issue
Block a user