From 457537a211215ee8502f33eef3e95f22855b7429 Mon Sep 17 00:00:00 2001 From: Richard Kovacs Date: Thu, 12 Jan 2023 14:57:38 +0100 Subject: [PATCH 1/5] Make tmp dir configurable and dynamic at local-up-cluster.sh --- hack/local-up-cluster.sh | 43 ++++++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/hack/local-up-cluster.sh b/hack/local-up-cluster.sh index 20355a5074d..5bf67017430 100755 --- a/hack/local-up-cluster.sh +++ b/hack/local-up-cluster.sh @@ -217,6 +217,7 @@ LOG_LEVEL=${LOG_LEVEL:-3} # Use to increase verbosity on particular files, e.g. LOG_SPEC=token_controller*=5,other_controller*=4 LOG_SPEC=${LOG_SPEC:-""} LOG_DIR=${LOG_DIR:-"/tmp"} +TMP_DIR=${TMP_DIR:-$(mktemp -d 2>/dev/null || mktemp -d -t test-cluster.XXXXXX)} CONTAINER_RUNTIME=${CONTAINER_RUNTIME:-"remote"} CONTAINER_RUNTIME_ENDPOINT=${CONTAINER_RUNTIME_ENDPOINT:-"unix:///run/containerd/containerd.sock"} RUNTIME_REQUEST_TIMEOUT=${RUNTIME_REQUEST_TIMEOUT:-"2m"} @@ -412,7 +413,7 @@ function start_etcd { function set_service_accounts { SERVICE_ACCOUNT_LOOKUP=${SERVICE_ACCOUNT_LOOKUP:-true} - SERVICE_ACCOUNT_KEY=${SERVICE_ACCOUNT_KEY:-/tmp/kube-serviceaccount.key} + SERVICE_ACCOUNT_KEY=${SERVICE_ACCOUNT_KEY:-${TMP_DIR}/kube-serviceaccount.key} # Generate ServiceAccount key if needed if [[ ! -f "${SERVICE_ACCOUNT_KEY}" ]]; then mkdir -p "$(dirname "${SERVICE_ACCOUNT_KEY}")" @@ -511,7 +512,7 @@ function start_apiserver { fi if [[ -z "${EGRESS_SELECTOR_CONFIG_FILE:-}" ]]; then - cat < /tmp/kube_egress_selector_configuration.yaml + cat < ${TMP_DIR}/kube_egress_selector_configuration.yaml apiVersion: apiserver.k8s.io/v1beta1 kind: EgressSelectorConfiguration egressSelections: @@ -525,18 +526,18 @@ egressSelections: connection: proxyProtocol: Direct EOF - EGRESS_SELECTOR_CONFIG_FILE="/tmp/kube_egress_selector_configuration.yaml" + EGRESS_SELECTOR_CONFIG_FILE="${TMP_DIR}/kube_egress_selector_configuration.yaml" fi if [[ -z "${AUDIT_POLICY_FILE}" ]]; then - cat < /tmp/kube-audit-policy-file + cat < ${TMP_DIR}/kube-audit-policy-file # Log all requests at the Metadata level. apiVersion: audit.k8s.io/v1 kind: Policy rules: - level: Metadata EOF - AUDIT_POLICY_FILE="/tmp/kube-audit-policy-file" + AUDIT_POLICY_FILE="${TMP_DIR}/kube-audit-policy-file" fi APISERVER_LOG=${LOG_DIR}/kube-apiserver.log @@ -734,7 +735,7 @@ function start_kubelet { generate_kubelet_certs fi - cat < /tmp/kubelet.yaml + cat < ${TMP_DIR}/kubelet.yaml apiVersion: kubelet.config.k8s.io/v1beta1 kind: KubeletConfiguration address: "${KUBELET_HOST}" @@ -755,7 +756,7 @@ resolvConf: "${KUBELET_RESOLV_CONF}" EOF if [[ "$FEATURE_GATES" == *KubeletTracing=true* ]]; then - cat <> /tmp/kubelet.yaml + cat <> ${TMP_DIR}/kubelet.yaml tracing: endpoint: localhost:4317 # the default value samplingRatePerMillion: 1000000 # sample always @@ -813,11 +814,11 @@ EOF if [[ -n ${FEATURE_GATES} ]]; then parse_feature_gates "${FEATURE_GATES}" fi - } >>/tmp/kubelet.yaml + } >>${TMP_DIR}/kubelet.yaml # shellcheck disable=SC2024 sudo -E "${GO_OUT}/kubelet" "${all_kubelet_flags[@]}" \ - --config=/tmp/kubelet.yaml >"${KUBELET_LOG}" 2>&1 & + --config=${TMP_DIR}/kubelet.yaml >"${KUBELET_LOG}" 2>&1 & KUBELET_PID=$! # Quick check that kubelet is running. @@ -837,7 +838,7 @@ function start_kubeproxy { wait_node_ready fi - cat < /tmp/kube-proxy.yaml + cat < ${TMP_DIR}/kube-proxy.yaml apiVersion: kubeproxy.config.k8s.io/v1alpha1 kind: KubeProxyConfiguration clientConnection: @@ -854,7 +855,7 @@ conntrack: EOF if [[ -n ${FEATURE_GATES} ]]; then parse_feature_gates "${FEATURE_GATES}" - fi >>/tmp/kube-proxy.yaml + fi >>${TMP_DIR}/kube-proxy.yaml if [[ "${REUSE_CERTS}" != true ]]; then generate_kubeproxy_certs @@ -863,7 +864,7 @@ EOF # shellcheck disable=SC2024 sudo "${GO_OUT}/kube-proxy" \ --v="${LOG_LEVEL}" \ - --config=/tmp/kube-proxy.yaml \ + --config=${TMP_DIR}/kube-proxy.yaml \ --master="https://${API_HOST}:${API_SECURE_PORT}" >"${PROXY_LOG}" 2>&1 & PROXY_PID=$! } @@ -871,7 +872,7 @@ EOF function start_kubescheduler { SCHEDULER_LOG=${LOG_DIR}/kube-scheduler.log - cat < /tmp/kube-scheduler.yaml + cat < ${TMP_DIR}/kube-scheduler.yaml apiVersion: kubescheduler.config.k8s.io/v1 kind: KubeSchedulerConfiguration clientConnection: @@ -881,7 +882,7 @@ leaderElection: EOF ${CONTROLPLANE_SUDO} "${GO_OUT}/kube-scheduler" \ --v="${LOG_LEVEL}" \ - --config=/tmp/kube-scheduler.yaml \ + --config=${TMP_DIR}/kube-scheduler.yaml \ --feature-gates="${FEATURE_GATES}" \ --authentication-kubeconfig "${CERT_DIR}"/scheduler.kubeconfig \ --authorization-kubeconfig "${CERT_DIR}"/scheduler.kubeconfig \ @@ -1030,13 +1031,13 @@ function parse_eviction { function install_cni { cni_plugin_sha=CNI_PLUGINS_${CNI_TARGETARCH^^}_SHA256SUM echo "Installing CNI plugin binaries ..." \ - && curl -sSL --retry 5 --output /tmp/cni."${CNI_TARGETARCH}".tgz "${CNI_PLUGINS_URL}" \ - && echo "${!cni_plugin_sha} /tmp/cni.${CNI_TARGETARCH}.tgz" | tee /tmp/cni.sha256 \ - && sha256sum --ignore-missing -c /tmp/cni.sha256 \ - && rm -f /tmp/cni.sha256 \ + && curl -sSL --retry 5 --output ${TMP_DIR}/cni."${CNI_TARGETARCH}".tgz "${CNI_PLUGINS_URL}" \ + && echo "${!cni_plugin_sha} ${TMP_DIR}/cni.${CNI_TARGETARCH}.tgz" | tee ${TMP_DIR}/cni.sha256 \ + && sha256sum --ignore-missing -c ${TMP_DIR}/cni.sha256 \ + && rm -f ${TMP_DIR}/cni.sha256 \ && sudo mkdir -p /opt/cni/bin \ - && sudo tar -C /opt/cni/bin -xzvf /tmp/cni."${CNI_TARGETARCH}".tgz \ - && rm -rf /tmp/cni."${CNI_TARGETARCH}".tgz \ + && sudo tar -C /opt/cni/bin -xzvf ${TMP_DIR}/cni."${CNI_TARGETARCH}".tgz \ + && rm -rf ${TMP_DIR}/cni."${CNI_TARGETARCH}".tgz \ && sudo find /opt/cni/bin -type f -not \( \ -iname host-local \ -o -iname bridge \ @@ -1139,7 +1140,7 @@ if [ "${GO_OUT}" == "" ]; then fi echo "Detected host and ready to start services. Doing some housekeeping first..." echo "Using GO_OUT ${GO_OUT}" -export KUBELET_CIDFILE=/tmp/kubelet.cid +export KUBELET_CIDFILE=${TMP_DIR}/kubelet.cid if [[ "${ENABLE_DAEMON}" = false ]]; then trap cleanup EXIT fi From 63982fc1fd4e2a3c59d9886a906a4ad416a5ab19 Mon Sep 17 00:00:00 2001 From: Richard Kovacs Date: Thu, 12 Jan 2023 18:06:14 +0100 Subject: [PATCH 2/5] Display generated config file locations on successful local-up-cluster.sh execution --- hack/local-up-cluster.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/hack/local-up-cluster.sh b/hack/local-up-cluster.sh index 5bf67017430..4eb033bdf53 100755 --- a/hack/local-up-cluster.sh +++ b/hack/local-up-cluster.sh @@ -957,9 +957,17 @@ if [[ "${START_MODE}" != "kubeletonly" ]]; then else echo "Local Kubernetes cluster is running." fi + + echo + echo "Configurations:" + for f in $(ls ${TMP_DIR}); do + echo " ${TMP_DIR}/${f}" + done + cat < Date: Wed, 18 Jan 2023 10:58:41 +0100 Subject: [PATCH 3/5] Use proper mktmp command in local-up-cluster.sh --- hack/local-up-cluster.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hack/local-up-cluster.sh b/hack/local-up-cluster.sh index 4eb033bdf53..7427d480062 100755 --- a/hack/local-up-cluster.sh +++ b/hack/local-up-cluster.sh @@ -217,7 +217,7 @@ LOG_LEVEL=${LOG_LEVEL:-3} # Use to increase verbosity on particular files, e.g. LOG_SPEC=token_controller*=5,other_controller*=4 LOG_SPEC=${LOG_SPEC:-""} LOG_DIR=${LOG_DIR:-"/tmp"} -TMP_DIR=${TMP_DIR:-$(mktemp -d 2>/dev/null || mktemp -d -t test-cluster.XXXXXX)} +TMP_DIR=${TMP_DIR:-$(kube::realpath "$(mktemp -d -t "$(basename "$0").XXXXXX")")} CONTAINER_RUNTIME=${CONTAINER_RUNTIME:-"remote"} CONTAINER_RUNTIME_ENDPOINT=${CONTAINER_RUNTIME_ENDPOINT:-"unix:///run/containerd/containerd.sock"} RUNTIME_REQUEST_TIMEOUT=${RUNTIME_REQUEST_TIMEOUT:-"2m"} From 89539e80dc231325344c9aa80d5921b351b33fcd Mon Sep 17 00:00:00 2001 From: Richard Kovacs Date: Wed, 18 Jan 2023 11:15:38 +0100 Subject: [PATCH 4/5] Fix lint errors at local-up-cluster.sh --- hack/local-up-cluster.sh | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/hack/local-up-cluster.sh b/hack/local-up-cluster.sh index 7427d480062..f76fc7339a2 100755 --- a/hack/local-up-cluster.sh +++ b/hack/local-up-cluster.sh @@ -512,7 +512,7 @@ function start_apiserver { fi if [[ -z "${EGRESS_SELECTOR_CONFIG_FILE:-}" ]]; then - cat < ${TMP_DIR}/kube_egress_selector_configuration.yaml + cat < "${TMP_DIR}"/kube_egress_selector_configuration.yaml apiVersion: apiserver.k8s.io/v1beta1 kind: EgressSelectorConfiguration egressSelections: @@ -530,7 +530,7 @@ EOF fi if [[ -z "${AUDIT_POLICY_FILE}" ]]; then - cat < ${TMP_DIR}/kube-audit-policy-file + cat < "${TMP_DIR}"/kube-audit-policy-file # Log all requests at the Metadata level. apiVersion: audit.k8s.io/v1 kind: Policy @@ -735,7 +735,7 @@ function start_kubelet { generate_kubelet_certs fi - cat < ${TMP_DIR}/kubelet.yaml + cat < "${TMP_DIR}"/kubelet.yaml apiVersion: kubelet.config.k8s.io/v1beta1 kind: KubeletConfiguration address: "${KUBELET_HOST}" @@ -756,7 +756,7 @@ resolvConf: "${KUBELET_RESOLV_CONF}" EOF if [[ "$FEATURE_GATES" == *KubeletTracing=true* ]]; then - cat <> ${TMP_DIR}/kubelet.yaml + cat <> "${TMP_DIR}"/kubelet.yaml tracing: endpoint: localhost:4317 # the default value samplingRatePerMillion: 1000000 # sample always @@ -814,11 +814,11 @@ EOF if [[ -n ${FEATURE_GATES} ]]; then parse_feature_gates "${FEATURE_GATES}" fi - } >>${TMP_DIR}/kubelet.yaml + } >>"${TMP_DIR}"/kubelet.yaml # shellcheck disable=SC2024 sudo -E "${GO_OUT}/kubelet" "${all_kubelet_flags[@]}" \ - --config=${TMP_DIR}/kubelet.yaml >"${KUBELET_LOG}" 2>&1 & + --config="${TMP_DIR}"/kubelet.yaml >"${KUBELET_LOG}" 2>&1 & KUBELET_PID=$! # Quick check that kubelet is running. @@ -838,7 +838,7 @@ function start_kubeproxy { wait_node_ready fi - cat < ${TMP_DIR}/kube-proxy.yaml + cat < "${TMP_DIR}"/kube-proxy.yaml apiVersion: kubeproxy.config.k8s.io/v1alpha1 kind: KubeProxyConfiguration clientConnection: @@ -855,7 +855,7 @@ conntrack: EOF if [[ -n ${FEATURE_GATES} ]]; then parse_feature_gates "${FEATURE_GATES}" - fi >>${TMP_DIR}/kube-proxy.yaml + fi >>"${TMP_DIR}"/kube-proxy.yaml if [[ "${REUSE_CERTS}" != true ]]; then generate_kubeproxy_certs @@ -864,7 +864,7 @@ EOF # shellcheck disable=SC2024 sudo "${GO_OUT}/kube-proxy" \ --v="${LOG_LEVEL}" \ - --config=${TMP_DIR}/kube-proxy.yaml \ + --config="${TMP_DIR}"/kube-proxy.yaml \ --master="https://${API_HOST}:${API_SECURE_PORT}" >"${PROXY_LOG}" 2>&1 & PROXY_PID=$! } @@ -872,7 +872,7 @@ EOF function start_kubescheduler { SCHEDULER_LOG=${LOG_DIR}/kube-scheduler.log - cat < ${TMP_DIR}/kube-scheduler.yaml + cat < "${TMP_DIR}"/kube-scheduler.yaml apiVersion: kubescheduler.config.k8s.io/v1 kind: KubeSchedulerConfiguration clientConnection: @@ -882,7 +882,7 @@ leaderElection: EOF ${CONTROLPLANE_SUDO} "${GO_OUT}/kube-scheduler" \ --v="${LOG_LEVEL}" \ - --config=${TMP_DIR}/kube-scheduler.yaml \ + --config="${TMP_DIR}"/kube-scheduler.yaml \ --feature-gates="${FEATURE_GATES}" \ --authentication-kubeconfig "${CERT_DIR}"/scheduler.kubeconfig \ --authorization-kubeconfig "${CERT_DIR}"/scheduler.kubeconfig \ @@ -960,8 +960,8 @@ if [[ "${START_MODE}" != "kubeletonly" ]]; then echo echo "Configurations:" - for f in $(ls ${TMP_DIR}); do - echo " ${TMP_DIR}/${f}" + for f in "${TMP_DIR}"/*; do + echo " ${f}" done cat < Date: Wed, 18 Jan 2023 12:20:33 +0100 Subject: [PATCH 5/5] Fix lint errors at local-up-cluster.sh --- hack/local-up-cluster.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hack/local-up-cluster.sh b/hack/local-up-cluster.sh index f76fc7339a2..16e8ed9a1cf 100755 --- a/hack/local-up-cluster.sh +++ b/hack/local-up-cluster.sh @@ -1040,7 +1040,7 @@ function install_cni { cni_plugin_sha=CNI_PLUGINS_${CNI_TARGETARCH^^}_SHA256SUM echo "Installing CNI plugin binaries ..." \ && curl -sSL --retry 5 --output "${TMP_DIR}"/cni."${CNI_TARGETARCH}".tgz "${CNI_PLUGINS_URL}" \ - && echo "${!cni_plugin_sha} "${TMP_DIR}"/cni.${CNI_TARGETARCH}.tgz" | tee "${TMP_DIR}"/cni.sha256 \ + && echo "${!cni_plugin_sha} ${TMP_DIR}/cni.${CNI_TARGETARCH}.tgz" | tee "${TMP_DIR}"/cni.sha256 \ && sha256sum --ignore-missing -c "${TMP_DIR}"/cni.sha256 \ && rm -f "${TMP_DIR}"/cni.sha256 \ && sudo mkdir -p /opt/cni/bin \