mirror of
				https://github.com/k3s-io/kubernetes.git
				synced 2025-11-04 07:49:35 +00:00 
			
		
		
		
	
		
			
				
	
	
		
			304 lines
		
	
	
		
			8.4 KiB
		
	
	
	
		
			Bash
		
	
	
		
			Executable File
		
	
	
	
	
			
		
		
	
	
			304 lines
		
	
	
		
			8.4 KiB
		
	
	
	
		
			Bash
		
	
	
		
			Executable File
		
	
	
	
	
#!/usr/bin/env bash
 | 
						|
 | 
						|
# Copyright 2015 The Kubernetes Authors.
 | 
						|
#
 | 
						|
# Licensed under the Apache License, Version 2.0 (the "License");
 | 
						|
# you may not use this file except in compliance with the License.
 | 
						|
# You may obtain a copy of the License at
 | 
						|
#
 | 
						|
#     http://www.apache.org/licenses/LICENSE-2.0
 | 
						|
#
 | 
						|
# Unless required by applicable law or agreed to in writing, software
 | 
						|
# distributed under the License is distributed on an "AS IS" BASIS,
 | 
						|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 | 
						|
# See the License for the specific language governing permissions and
 | 
						|
# limitations under the License.
 | 
						|
 | 
						|
# This script grabs profiles from running components.
 | 
						|
# Usage: `hack/grab-profiles.sh`.
 | 
						|
 | 
						|
set -o errexit
 | 
						|
set -o nounset
 | 
						|
set -o pipefail
 | 
						|
 | 
						|
function grab_profiles_from_component {
 | 
						|
  local requested_profiles=$1
 | 
						|
  local mem_pprof_flags=$2
 | 
						|
  local binary=$3
 | 
						|
  local tunnel_port=$4
 | 
						|
  local path=$5
 | 
						|
  local output_prefix=$6
 | 
						|
  local timestamp=$7
 | 
						|
 | 
						|
  echo "binary: $binary"
 | 
						|
 | 
						|
  for profile in ${requested_profiles}; do
 | 
						|
    case ${profile} in
 | 
						|
      cpu)
 | 
						|
        go tool pprof "-pdf" "${binary}" "http://localhost:${tunnel_port}${path}/debug/pprof/profile" > "${output_prefix}-${profile}-profile-${timestamp}.pdf"
 | 
						|
        ;;
 | 
						|
      mem)
 | 
						|
        # There are different kinds of memory profiles that are available that
 | 
						|
        # had to be grabbed separately: --inuse-space, --inuse-objects,
 | 
						|
        # --alloc-space, --alloc-objects. We need to iterate over all requested
 | 
						|
        # kinds.
 | 
						|
        for flag in ${mem_pprof_flags}; do
 | 
						|
          go tool pprof "-${flag}" "-pdf" "${binary}" "http://localhost:${tunnel_port}${path}/debug/pprof/heap" > "${output_prefix}-${profile}-${flag}-profile-${timestamp}.pdf"
 | 
						|
        done
 | 
						|
        ;;
 | 
						|
    esac
 | 
						|
  done
 | 
						|
}
 | 
						|
 | 
						|
KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
 | 
						|
source "${KUBE_ROOT}/hack/lib/init.sh"
 | 
						|
 | 
						|
server_addr=""
 | 
						|
kubelet_addresses=""
 | 
						|
kubelet_binary=""
 | 
						|
master_binary=""
 | 
						|
scheduler_binary=""
 | 
						|
scheduler_port="10251"
 | 
						|
controller_manager_port="10252"
 | 
						|
controller_manager_binary=""
 | 
						|
requested_profiles=""
 | 
						|
mem_pprof_flags=""
 | 
						|
profile_components=""
 | 
						|
output_dir="."
 | 
						|
tunnel_port="${tunnel_port:-1234}"
 | 
						|
 | 
						|
args=$(getopt -o s:mho:k:c -l server:,master,heapster,output:,kubelet:,scheduler,controller-manager,help,inuse-space,inuse-objects,alloc-space,alloc-objects,cpu,kubelet-binary:,master-binary:,scheduler-binary:,controller-manager-binary:,scheduler-port:,controller-manager-port: -- "$@")
 | 
						|
if [[ $? ]]; then
 | 
						|
  >&2 echo "Error in getopt"
 | 
						|
  exit 1
 | 
						|
fi
 | 
						|
 | 
						|
HEAPSTER_VERSION="v0.18.2"
 | 
						|
MASTER_PPROF_PATH=""
 | 
						|
HEAPSTER_PPROF_PATH="/api/v1/namespaces/kube-system/services/monitoring-heapster/proxy"
 | 
						|
KUBELET_PPROF_PATH_PREFIX="/api/v1/proxy/nodes"
 | 
						|
SCHEDULER_PPROF_PATH_PREFIX="/api/v1/namespaces/kube-system/pods/kube-scheduler/proxy"
 | 
						|
CONTROLLER_MANAGER_PPROF_PATH_PREFIX="/api/v1/namespaces/kube-system/pods/kube-controller-manager/proxy"
 | 
						|
 | 
						|
eval set -- "${args}"
 | 
						|
 | 
						|
while true; do
 | 
						|
  case $1 in
 | 
						|
    -s|--server)
 | 
						|
      shift
 | 
						|
      if [ -z "$1" ]; then
 | 
						|
        >&2 echo "empty argument to --server flag"
 | 
						|
        exit 1
 | 
						|
      fi
 | 
						|
      server_addr=$1
 | 
						|
      shift
 | 
						|
      ;;
 | 
						|
    -m|--master)
 | 
						|
      shift
 | 
						|
      profile_components="master ${profile_components}"
 | 
						|
      ;;
 | 
						|
    --master-binary)
 | 
						|
      shift
 | 
						|
      if [ -z "$1" ]; then
 | 
						|
        >&2 echo "empty argument to --master-binary flag"
 | 
						|
        exit 1
 | 
						|
      fi
 | 
						|
      master_binary=$1
 | 
						|
      shift
 | 
						|
      ;;
 | 
						|
    -h|--heapster)
 | 
						|
      shift
 | 
						|
      profile_components="heapster ${profile_components}"
 | 
						|
      ;;
 | 
						|
    -k|--kubelet)
 | 
						|
      shift
 | 
						|
      profile_components="kubelet ${profile_components}"
 | 
						|
      if [ -z "$1" ]; then
 | 
						|
        >&2 echo "empty argument to --kubelet flag"
 | 
						|
        exit 1
 | 
						|
      fi
 | 
						|
      kubelet_addresses="$1 $kubelet_addresses"
 | 
						|
      shift
 | 
						|
      ;;
 | 
						|
    --kubelet-binary)
 | 
						|
      shift
 | 
						|
      if [ -z "$1" ]; then
 | 
						|
        >&2 echo "empty argument to --kubelet-binary flag"
 | 
						|
        exit 1
 | 
						|
      fi
 | 
						|
      kubelet_binary=$1
 | 
						|
      shift
 | 
						|
      ;;
 | 
						|
    --scheduler)
 | 
						|
      shift
 | 
						|
      profile_components="scheduler ${profile_components}"
 | 
						|
      ;;
 | 
						|
    --scheduler-binary)
 | 
						|
      shift
 | 
						|
      if [ -z "$1" ]; then
 | 
						|
        >&2 echo "empty argument to --scheduler-binary flag"
 | 
						|
        exit 1
 | 
						|
      fi
 | 
						|
      scheduler_binary=$1
 | 
						|
      shift
 | 
						|
      ;;
 | 
						|
    --scheduler-port)
 | 
						|
      shift
 | 
						|
      if [ -z "$1" ]; then
 | 
						|
        >&2 echo "empty argument to --scheduler-port flag"
 | 
						|
        exit 1
 | 
						|
      fi
 | 
						|
      scheduler_port=$1
 | 
						|
      shift
 | 
						|
      ;;
 | 
						|
    -c|--controller-manager)
 | 
						|
      shift
 | 
						|
      profile_components="controller-manager ${profile_components}"
 | 
						|
      ;;
 | 
						|
    --controller-manager-binary)
 | 
						|
      shift
 | 
						|
      if [ -z "$1" ]; then
 | 
						|
        >&2 echo "empty argument to --controller-manager-binary flag"
 | 
						|
        exit 1
 | 
						|
      fi
 | 
						|
      controller_manager_binary=$1
 | 
						|
      shift
 | 
						|
      ;;
 | 
						|
    --controller-manager-port)
 | 
						|
      shift
 | 
						|
      if [ -z "$1" ]; then
 | 
						|
        >&2 echo "empty argument to --controller-manager-port flag"
 | 
						|
        exit 1
 | 
						|
      fi
 | 
						|
      controller_manager_port=$1
 | 
						|
      shift
 | 
						|
      ;;
 | 
						|
    -o|--output)
 | 
						|
      shift
 | 
						|
      if [ -z "$1" ]; then
 | 
						|
        >&2 echo "empty argument to --output flag"
 | 
						|
        exit 1
 | 
						|
      fi
 | 
						|
      output_dir=$1
 | 
						|
      shift
 | 
						|
      ;;
 | 
						|
    --inuse-space)
 | 
						|
      shift
 | 
						|
      requested_profiles="mem ${requested_profiles}"
 | 
						|
      mem_pprof_flags="inuse_space ${mem_pprof_flags}"
 | 
						|
      ;;
 | 
						|
    --inuse-objects)
 | 
						|
      shift
 | 
						|
      requested_profiles="mem ${requested_profiles}"
 | 
						|
      mem_pprof_flags="inuse_objects ${mem_pprof_flags}"
 | 
						|
      ;;
 | 
						|
    --alloc-space)
 | 
						|
      shift
 | 
						|
      requested_profiles="mem ${requested_profiles}"
 | 
						|
      mem_pprof_flags="alloc_space ${mem_pprof_flags}"
 | 
						|
      ;;
 | 
						|
    --alloc-objects)
 | 
						|
      shift
 | 
						|
      requested_profiles="mem ${requested_profiles}"
 | 
						|
      mem_pprof_flags="alloc_objects ${mem_pprof_flags}"
 | 
						|
      ;;
 | 
						|
    --cpu)
 | 
						|
      shift
 | 
						|
      requested_profiles="cpu ${requested_profiles}"
 | 
						|
      ;;
 | 
						|
    --help)
 | 
						|
      shift
 | 
						|
      echo "Recognized options:
 | 
						|
        -o/--output,
 | 
						|
        -s/--server,
 | 
						|
        -m/--master,
 | 
						|
        -h/--heapster,
 | 
						|
        --inuse-space,
 | 
						|
        --inuse-objects,
 | 
						|
        --alloc-space,
 | 
						|
        --alloc-objects,
 | 
						|
        --cpu,
 | 
						|
        --help"
 | 
						|
      exit 0
 | 
						|
      ;;
 | 
						|
    --)
 | 
						|
      shift
 | 
						|
      break;
 | 
						|
      ;;
 | 
						|
  esac
 | 
						|
done
 | 
						|
 | 
						|
if [[ -z "${server_addr}" ]]; then
 | 
						|
  >&2 echo "Server flag is required"
 | 
						|
  exit 1
 | 
						|
fi
 | 
						|
 | 
						|
if [[ -z "${profile_components}" ]]; then
 | 
						|
  >&2 echo "Choose at least one component to profile"
 | 
						|
  exit 1
 | 
						|
fi
 | 
						|
 | 
						|
if [[ -z "${requested_profiles}" ]]; then
 | 
						|
  >&2 echo "Choose at least one profiling option"
 | 
						|
  exit 1
 | 
						|
fi
 | 
						|
 | 
						|
gcloud compute ssh "${server_addr}" --ssh-flag=-nN --ssh-flag=-L"${tunnel_port}":localhost:8080 &
 | 
						|
 | 
						|
echo "Waiting for tunnel to be created..."
 | 
						|
kube::util::wait_for_url http://localhost:"${tunnel_port}"/healthz
 | 
						|
 | 
						|
SSH_PID=$(pgrep -f "/usr/bin/ssh.*${tunnel_port}:localhost:8080")
 | 
						|
kube::util::trap_add "kill $SSH_PID" EXIT
 | 
						|
kube::util::trap_add "kill $SSH_PID" SIGTERM
 | 
						|
 | 
						|
requested_profiles=$(echo "${requested_profiles}" | xargs -n1 | LC_ALL=C sort -u | xargs)
 | 
						|
profile_components=$(echo "${profile_components}" | xargs -n1 | LC_ALL=C sort -u | xargs)
 | 
						|
kubelet_addresses=$(echo "${kubelet_addresses}" | xargs -n1 | LC_ALL=C sort -u | xargs)
 | 
						|
echo "requested profiles: ${requested_profiles}"
 | 
						|
echo "flags for heap profile: ${mem_pprof_flags}"
 | 
						|
 | 
						|
timestamp=$(date +%Y%m%d%H%M%S)
 | 
						|
binary=""
 | 
						|
 | 
						|
for component in ${profile_components}; do
 | 
						|
  case ${component} in
 | 
						|
    master)
 | 
						|
      path=${MASTER_PPROF_PATH}
 | 
						|
      binary=${master_binary}
 | 
						|
      ;;
 | 
						|
    controller-manager)
 | 
						|
      path="${CONTROLLER_MANAGER_PPROF_PATH_PREFIX}-${server_addr}:${controller_manager_port}"
 | 
						|
      binary=${controller_manager_binary}
 | 
						|
      ;;
 | 
						|
    scheduler)
 | 
						|
      path="${SCHEDULER_PPROF_PATH_PREFIX}-${server_addr}:${scheduler_port}"
 | 
						|
      binary=${scheduler_binary}
 | 
						|
      ;;
 | 
						|
    heapster)
 | 
						|
      rm heapster
 | 
						|
      wget https://github.com/kubernetes/heapster/releases/download/${HEAPSTER_VERSION}/heapster
 | 
						|
      kube::util::trap_add 'rm -f heapster' EXIT
 | 
						|
      kube::util::trap_add 'rm -f heapster' SIGTERM
 | 
						|
      binary=heapster
 | 
						|
      path=${HEAPSTER_PPROF_PATH}
 | 
						|
      ;;
 | 
						|
    kubelet)
 | 
						|
      path="${KUBELET_PPROF_PATH_PREFIX}"
 | 
						|
      if [[ -z "${kubelet_binary}" ]]; then
 | 
						|
        binary="${KUBE_ROOT}/_output/local/bin/linux/amd64/kubelet"
 | 
						|
      else
 | 
						|
        binary=${kubelet_binary}
 | 
						|
      fi
 | 
						|
      ;;
 | 
						|
  esac
 | 
						|
 | 
						|
  if [[ "${component}" == "kubelet" ]]; then
 | 
						|
    for node in ${kubelet_addresses//[,;]/' '}; do
 | 
						|
      grab_profiles_from_component "${requested_profiles}" "${mem_pprof_flags}" "${binary}" "${tunnel_port}" "${path}/${node}/proxy" "${output_dir}/${component}" "${timestamp}"
 | 
						|
    done
 | 
						|
  else
 | 
						|
    grab_profiles_from_component "${requested_profiles}" "${mem_pprof_flags}" "${binary}" "${tunnel_port}" "${path}" "${output_dir}/${component}" "${timestamp}"
 | 
						|
  fi
 | 
						|
done
 |