github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-09-14 05:36:12 +00:00
Code Issues Projects Releases Wiki Activity

cluster/gce: set KUBE_LIST_FROM_CACHE_INCONSISTENCY_DETECTOR when requested by a CI jobs

Browse Source 
When the KUBE_LIST_FROM_CACHE_INCONSISTENCY_DETECTOR environment variable was set
then clients (client-go, dynamic-client) perform a data consistency check
for requests that have a high chance of being served from the watch-cache.

The consistency check is meant to be enforced only in the CI, not in production.
The check ensures that data retrieved by a list api call from the watch-cache
is exactly the same as data received by the list api call from etcd.

Note that this function will panic when data inconsistency is detected.
 This is intentional because we want to catch it in the CI.

Note that the new env var is set in the same places
the ENABLE_CACHE_MUTATION_DETECTOR is set.
This commit is contained in:
Lukasz Szaszkiewicz
2024-06-07 13:08:30 +02:00
parent 3532601ad8
commit 1aa6797b74
4 changed files with 32 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
cluster/gce/gci/configure-helper.sh
View File

@@ -1806,6 +1806,15 @@ function prepare-kube-proxy-manifest-variables {
kube_watchlist_inconsistency_detector_env_name="- name: KUBE_WATCHLIST_INCONSISTENCY_DETECTOR"
kube_watchlist_inconsistency_detector_env_value="value: \"${ENABLE_KUBE_WATCHLIST_INCONSISTENCY_DETECTOR}\""
fi
local kube_list_from_cache_inconsistency_detector_env_name=""
local kube_list_from_cache_inconsistency_detector_env_value=""
if [[ -n "${ENABLE_KUBE_LIST_FROM_CACHE_INCONSISTENCY_DETECTOR:-}" ]]; then
if [[ -z "${container_env}" ]]; then
container_env="env:"
fi
kube_list_from_cache_inconsistency_detector_env_name="- name: KUBE_LIST_FROM_CACHE_INCONSISTENCY_DETECTOR"
kube_list_from_cache_inconsistency_detector_env_value="value: \"${ENABLE_KUBE_LIST_FROM_CACHE_INCONSISTENCY_DETECTOR}\""
fi
sed -i -e "s@{{kubeconfig}}@${kubeconfig}@g" "${src_file}"
sed -i -e "s@{{pillar\['kube_docker_registry'\]}}@${kube_docker_registry}@g" "${src_file}"
sed -i -e "s@{{pillar\['kube-proxy_docker_tag'\]}}@${kube_proxy_docker_tag}@g" "${src_file}"
@@ -1817,6 +1826,8 @@ function prepare-kube-proxy-manifest-variables {
sed -i -e "s@{{kube_cache_mutation_detector_env_value}}@${kube_cache_mutation_detector_env_value}@g" "${src_file}"
sed -i -e "s@{{kube_watchlist_inconsistency_detector_env_name}}@${kube_watchlist_inconsistency_detector_env_name}@g" "${src_file}"
sed -i -e "s@{{kube_watchlist_inconsistency_detector_env_value}}@${kube_watchlist_inconsistency_detector_env_value}@g" "${src_file}"
sed -i -e "s@{{kube_list_from_cache_inconsistency_detector_env_name}}@${kube_list_from_cache_inconsistency_detector_env_name}@g" "${src_file}"
sed -i -e "s@{{kube_list_from_cache_inconsistency_detector_env_value}}@${kube_list_from_cache_inconsistency_detector_env_value}@g" "${src_file}"
sed -i -e "s@{{ cpurequest }}@${KUBE_PROXY_CPU_REQUEST:-100m}@g" "${src_file}"
sed -i -e "s@{{ memoryrequest }}@${KUBE_PROXY_MEMORY_REQUEST:-50Mi}@g" "${src_file}"
sed -i -e "s@{{api_servers_with_port}}@${api_servers}@g" "${src_file}"
@@ -2255,10 +2266,15 @@ function start-kube-controller-manager {
fi
container_env+="{\"name\": \"KUBE_WATCHLIST_INCONSISTENCY_DETECTOR\", \"value\": \"${ENABLE_KUBE_WATCHLIST_INCONSISTENCY_DETECTOR}\"}"
fi
if [[ -n "${ENABLE_KUBE_LIST_FROM_CACHE_INCONSISTENCY_DETECTOR:-}" ]]; then
if [[ -n "${container_env}" ]]; then
container_env="${container_env}, "
fi
container_env+="{\"name\": \"KUBE_LIST_FROM_CACHE_INCONSISTENCY_DETECTOR\", \"value\": \"${ENABLE_KUBE_LIST_FROM_CACHE_INCONSISTENCY_DETECTOR}\"}"
fi
if [[ -n "${container_env}" ]]; then
container_env="\"env\":[${container_env}],"
fi
local paramstring
paramstring="$(convert-manifest-params "${params[*]}")"
local -r src_file="${KUBE_HOME}/kube-manifests/kubernetes/gci-trusty/kube-controller-manager.manifest"
@@ -2370,6 +2386,12 @@ function start-cloud-controller-manager {
fi
container_env+="{\"name\": \"KUBE_WATCHLIST_INCONSISTENCY_DETECTOR\", \"value\": \"${ENABLE_KUBE_WATCHLIST_INCONSISTENCY_DETECTOR}\"}"
fi
if [[ -n "${ENABLE_KUBE_LIST_FROM_CACHE_INCONSISTENCY_DETECTOR:-}" ]]; then
if [[ -n "${container_env}" ]]; then
container_env="${container_env}, "
fi
container_env+="{\"name\": \"KUBE_LIST_FROM_CACHE_INCONSISTENCY_DETECTOR\", \"value\": \"${ENABLE_KUBE_LIST_FROM_CACHE_INCONSISTENCY_DETECTOR}\"}"
fi
if [[ -n "${container_env}" ]]; then
container_env="\"env\":[${container_env}],"
fi

6
cluster/gce/gci/configure-kubeapiserver.sh
View File

@@ -343,6 +343,12 @@ function start-kube-apiserver {
fi
container_env+="{\"name\": \"KUBE_WATCHLIST_INCONSISTENCY_DETECTOR\", \"value\": \"${ENABLE_KUBE_WATCHLIST_INCONSISTENCY_DETECTOR}\"}"
fi
if [[ -n "${ENABLE_KUBE_LIST_FROM_CACHE_INCONSISTENCY_DETECTOR:-}" ]]; then
if [[ -n "${container_env}" ]]; then
container_env="${container_env}, "
fi
container_env+="{\"name\": \"KUBE_LIST_FROM_CACHE_INCONSISTENCY_DETECTOR\", \"value\": \"${ENABLE_KUBE_LIST_FROM_CACHE_INCONSISTENCY_DETECTOR}\"}"
fi
if [[ -n "${ENABLE_PATCH_CONVERSION_DETECTOR:-}" ]]; then
if [[ -n "${container_env}" ]]; then
container_env="${container_env}, "

2
cluster/gce/manifests/kube-proxy.manifest
View File

@@ -33,6 +33,8 @@ spec:
{{kube_cache_mutation_detector_env_value}}
{{kube_watchlist_inconsistency_detector_env_name}}
{{kube_watchlist_inconsistency_detector_env_value}}
{{kube_list_from_cache_inconsistency_detector_env_name}}
{{kube_list_from_cache_inconsistency_detector_env_value}}
securityContext:
privileged: true
volumeMounts:

1
cluster/gce/util.sh
View File

@@ -1178,6 +1178,7 @@ ENABLE_APISERVER_ADVANCED_AUDIT: $(yaml-quote "${ENABLE_APISERVER_ADVANCED_AUDIT
ENABLE_APISERVER_DYNAMIC_AUDIT: $(yaml-quote "${ENABLE_APISERVER_DYNAMIC_AUDIT:-}")
ENABLE_CACHE_MUTATION_DETECTOR: $(yaml-quote "${ENABLE_CACHE_MUTATION_DETECTOR:-false}")
ENABLE_KUBE_WATCHLIST_INCONSISTENCY_DETECTOR: $(yaml-quote "${ENABLE_KUBE_WATCHLIST_INCONSISTENCY_DETECTOR:-false}")
ENABLE_KUBE_LIST_FROM_CACHE_INCONSISTENCY_DETECTOR: $(yaml-quote "${ENABLE_KUBE_LIST_FROM_CACHE_INCONSISTENCY_DETECTOR:-false}")
ENABLE_PATCH_CONVERSION_DETECTOR: $(yaml-quote "${ENABLE_PATCH_CONVERSION_DETECTOR:-false}")
ADVANCED_AUDIT_POLICY: $(yaml-quote "${ADVANCED_AUDIT_POLICY:-}")
ADVANCED_AUDIT_BACKEND: $(yaml-quote "${ADVANCED_AUDIT_BACKEND:-log}")