From 99156b5bdcce2f2b06322e5d21518a7ca4d3fdcb Mon Sep 17 00:00:00 2001 From: Walter Fender Date: Fri, 21 Oct 2022 14:36:05 -0700 Subject: [PATCH] Turn on CCM in kube-up when cloudprovider=external Currently if we disable cloud provider by the following CLOUD_PROVIDER_FLAG=external KUBE_FEATURE_GATES=DisableCloudProviders=true,DisableKubeletCloudCredentialProviders=true we can no longer schedule workloads due to taints and a lack of node configuration. This pulls a CCM image from K/cloud-provider-gcp to run tests. This is a pre-step for taking the above feature gates to beta. It does not address the last known good dependency issue. Specifically the CCM image is built on top of client-go and staging. However this image will be an "old" verison of those libraries. So it does not test if those libraries work in the CCM. Fix shellcheck errors. Add CCM_FEATURE_GATES for testing. Switching to extended regex from perl regex. Adding instrumentation to cluster configuration. Improved regex to not greedily get key-value pairs. Fixed issue with error on regex no line match. Switch credentialprovider version to v1alpha1 --- build/lib/release.sh | 1 + .../cloud-node-controller-binding.yaml | 46 ++++ .../cloud-node-controller-role.yaml | 212 ++++++++++++++++++ .../pvl-controller-role.yaml | 23 ++ cluster/gce/config-default.sh | 3 + cluster/gce/config-test.sh | 3 + cluster/gce/gci/configure-helper.sh | 117 ++++++++++ cluster/gce/gci/configure.sh | 2 +- .../cloud-controller-manager.manifest | 113 ++++++++++ cluster/gce/util.sh | 1 + 10 files changed, 520 insertions(+), 1 deletion(-) create mode 100644 cluster/addons/cloud-controller-manager/cloud-node-controller-binding.yaml create mode 100644 cluster/addons/cloud-controller-manager/cloud-node-controller-role.yaml create mode 100644 cluster/addons/cloud-controller-manager/pvl-controller-role.yaml create mode 100644 cluster/gce/manifests/cloud-controller-manager.manifest diff --git a/build/lib/release.sh b/build/lib/release.sh index 1ebceb27728..4a8a272c0b0 100644 --- a/build/lib/release.sh +++ b/build/lib/release.sh @@ -442,6 +442,7 @@ function kube::release::package_kube_manifests_tarball() { cp "${src_dir}/kube-apiserver.manifest" "${dst_dir}" cp "${src_dir}/konnectivity-server.yaml" "${dst_dir}" cp "${src_dir}/abac-authz-policy.jsonl" "${dst_dir}" + cp "${src_dir}/cloud-controller-manager.manifest" "${dst_dir}" cp "${src_dir}/kube-controller-manager.manifest" "${dst_dir}" cp "${src_dir}/kube-addon-manager.yaml" "${dst_dir}" cp "${src_dir}/glbc.manifest" "${dst_dir}" diff --git a/cluster/addons/cloud-controller-manager/cloud-node-controller-binding.yaml b/cluster/addons/cloud-controller-manager/cloud-node-controller-binding.yaml new file mode 100644 index 00000000000..4be96052de1 --- /dev/null +++ b/cluster/addons/cloud-controller-manager/cloud-node-controller-binding.yaml @@ -0,0 +1,46 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + addonmanager.kubernetes.io/mode: Reconcile + name: system::leader-locking-cloud-controller-manager + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: system::leader-locking-cloud-controller-manager +subjects: +- kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + addonmanager.kubernetes.io/mode: Reconcile + name: system:cloud-controller-manager +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager +subjects: +- kind: User + apiGroup: rbac.authorization.k8s.io + name: system:cloud-controller-manager +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + addonmanager.kubernetes.io/mode: Reconcile + name: system:controller:cloud-node-controller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:controller:cloud-node-controller +subjects: +- kind: ServiceAccount + name: cloud-node-controller + namespace: kube-system + diff --git a/cluster/addons/cloud-controller-manager/cloud-node-controller-role.yaml b/cluster/addons/cloud-controller-manager/cloud-node-controller-role.yaml new file mode 100644 index 00000000000..23e032d92e3 --- /dev/null +++ b/cluster/addons/cloud-controller-manager/cloud-node-controller-role.yaml @@ -0,0 +1,212 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + addonmanager.kubernetes.io/mode: Reconcile + name: system:cloud-controller-manager +rules: +- apiGroups: + - "" + - events.k8s.io + resources: + - events + verbs: + - create + - patch + - update +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create +- apiGroups: + - coordination.k8s.io + resourceNames: + - cloud-controller-manager + resources: + - leases + verbs: + - get + - update +- apiGroups: + - "" + resources: + - endpoints + - serviceaccounts + verbs: + - create + - get + - update +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - update +- apiGroups: + - "" + resources: + - namespaces + verbs: + - get +- apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - update +- apiGroups: + - "" + resources: + - secrets + verbs: + - create + - delete + - get + - update +- apiGroups: + - "authentication.k8s.io" + resources: + - tokenreviews + verbs: + - create +- apiGroups: + - "*" + resources: + - "*" + verbs: + - list + - watch +- apiGroups: + - "" + resources: + - serviceaccounts/token + verbs: + - create +- apiGroups: + - authentication.k8s.io + resources: + - subjectaccessreviews + verbs: + - create +- apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create +- apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create +- apiGroups: + - "" + resources: + - namespaces + - configmaps + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + addonmanager.kubernetes.io/mode: Reconcile + name: system:cloud-controller-manager + namespace: kube-system +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - watch +- apiGroups: + - "" + resources: + - configmaps + resourceNames: + - cloud-controller-manager + verbs: + - get + - update +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + addonmanager.kubernetes.io/mode: Reconcile + name: system::leader-locking-cloud-controller-manager + namespace: kube-system +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - watch +- apiGroups: + - "" + resources: + - configmaps + resourceNames: + - cloud-controller-manager + verbs: + - get + - update +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + addonmanager.kubernetes.io/mode: Reconcile + name: system:controller:cloud-node-controller +rules: +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - update + - delete + - patch +- apiGroups: + - "" + resources: + - nodes/status + verbs: + - get + - list + - update + - delete + - patch + +- apiGroups: + - "" + resources: + - pods + verbs: + - list + - delete +- apiGroups: + - "" + resources: + - pods/status + verbs: + - list + - delete diff --git a/cluster/addons/cloud-controller-manager/pvl-controller-role.yaml b/cluster/addons/cloud-controller-manager/pvl-controller-role.yaml new file mode 100644 index 00000000000..fb3306655c7 --- /dev/null +++ b/cluster/addons/cloud-controller-manager/pvl-controller-role.yaml @@ -0,0 +1,23 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + addonmanager.kubernetes.io/mode: Reconcile + name: system:controller:pvl-controller +rules: +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update +- apiGroups: + - "" + resources: + - persistentvolumeclaims + - persistentvolumes + verbs: + - list + - watch diff --git a/cluster/gce/config-default.sh b/cluster/gce/config-default.sh index 99564fbad17..cf4bd382bc4 100755 --- a/cluster/gce/config-default.sh +++ b/cluster/gce/config-default.sh @@ -255,6 +255,9 @@ if [[ (( "${KUBE_FEATURE_GATES:-}" == *"AllAlpha=true"* ) || ( "${KUBE_FEATURE_G RUN_CONTROLLERS="${RUN_CONTROLLERS:-*,endpointslice}" fi +# List of the set of feature gates recognized by the GCP CCM +export CCM_FEATURE_GATES="APIListChunking,APIPriorityAndFairness,APIResponseCompression,APIServerIdentity,APIServerTracing,AllAlpha,AllBeta,CustomResourceValidationExpressions,KMSv2,OpenAPIEnums,OpenAPIV3,RemainingItemCount,ServerSideFieldValidation,StorageVersionAPI,StorageVersionHash" + # Optional: set feature gates # shellcheck disable=SC2034 # Variables sourced in other scripts. FEATURE_GATES="${KUBE_FEATURE_GATES:-}" diff --git a/cluster/gce/config-test.sh b/cluster/gce/config-test.sh index 24b03d37a66..a6ba4abb15d 100755 --- a/cluster/gce/config-test.sh +++ b/cluster/gce/config-test.sh @@ -311,6 +311,9 @@ if [[ -n "${NODE_ACCELERATORS}" ]]; then fi fi +# List of the set of feature gates recognized by the GCP CCM +export CCM_FEATURE_GATES="APIListChunking,APIPriorityAndFairness,APIResponseCompression,APIServerIdentity,APIServerTracing,AllAlpha,AllBeta,CustomResourceValidationExpressions,KMSv2,OpenAPIEnums,OpenAPIV3,RemainingItemCount,ServerSideFieldValidation,StorageVersionAPI,StorageVersionHash" + # Optional: Install cluster DNS. # Set CLUSTER_DNS_CORE_DNS to 'false' to install kube-dns instead of CoreDNS. CLUSTER_DNS_CORE_DNS=${CLUSTER_DNS_CORE_DNS:-true} diff --git a/cluster/gce/gci/configure-helper.sh b/cluster/gce/gci/configure-helper.sh index f2311c8bbc9..d8ac2673a5d 100644 --- a/cluster/gce/gci/configure-helper.sh +++ b/cluster/gce/gci/configure-helper.sh @@ -800,6 +800,9 @@ function create-master-auth { if [[ -n "${KUBE_BOOTSTRAP_TOKEN:-}" ]]; then append_or_replace_prefixed_line "${known_tokens_csv}" "${KUBE_BOOTSTRAP_TOKEN}," "gcp:kube-bootstrap,uid:gcp:kube-bootstrap,system:masters" fi + if [[ -n "${CLOUD_CONTROLLER_MANAGER_TOKEN:-}" ]]; then + append_or_replace_prefixed_line "${known_tokens_csv}" "${CLOUD_CONTROLLER_MANAGER_TOKEN}," "system:cloud-controller-manager,uid:system:cloud-controller-manager" + fi if [[ -n "${KUBE_CONTROLLER_MANAGER_TOKEN:-}" ]]; then append_or_replace_prefixed_line "${known_tokens_csv}" "${KUBE_CONTROLLER_MANAGER_TOKEN}," "system:kube-controller-manager,uid:system:kube-controller-manager" fi @@ -1161,6 +1164,7 @@ rules: - level: None users: - system:kube-controller-manager + - system:cloud-controller-manager - system:kube-scheduler - system:serviceaccount:kube-system:endpoint-controller verbs: ["get", "update"] @@ -1185,6 +1189,7 @@ rules: - level: None users: - system:kube-controller-manager + - system:cloud-controller-manager verbs: ["get", "list"] resources: - group: "metrics.k8s.io" @@ -2241,6 +2246,112 @@ function start-kube-controller-manager { cp "${src_file}" /etc/kubernetes/manifests } +# (TODO/cloud-provider-gcp): Figure out how to inject +# Starts cloud controller manager. +# It prepares the log file, loads the docker image, calculates variables, sets them +# in the manifest file, and then copies the manifest file to /etc/kubernetes/manifests. +# +# Assumed vars (which are calculated in function compute-master-manifest-variables) +# CLOUD_CONFIG_OPT +# CLOUD_CONFIG_VOLUME +# CLOUD_CONFIG_MOUNT +# DOCKER_REGISTRY +function start-cloud-controller-manager { + echo "Start cloud provider controller-manager" + setup-addon-manifests "addons" "cloud-controller-manager" + + create-kubeconfig "cloud-controller-manager" "${CLOUD_CONTROLLER_MANAGER_TOKEN}" + echo "Preparing cloud provider controller-manager log file" + prepare-log-file /var/log/cloud-controller-manager.log "${CLOUD_CONTROLLER_MANAGER_RUNASUSER:-0}" + # Calculate variables and assemble the command line. + local params=("${CONTROLLER_MANAGER_TEST_LOG_LEVEL:-"--v=4"}" "${CONTROLLER_MANAGER_TEST_ARGS:-}" "${CLOUD_CONFIG_OPT}") + params+=("--secure-port=10258") + params+=("--use-service-account-credentials") + params+=("--cloud-provider=gce") + params+=("--kubeconfig=/etc/srv/kubernetes/cloud-controller-manager/kubeconfig") + params+=("--authorization-kubeconfig=/etc/srv/kubernetes/cloud-controller-manager/kubeconfig") + params+=("--authentication-kubeconfig=/etc/srv/kubernetes/cloud-controller-manager/kubeconfig") + if [[ -n "${INSTANCE_PREFIX:-}" ]]; then + params+=("--cluster-name=${INSTANCE_PREFIX}") + fi + if [[ -n "${CLUSTER_IP_RANGE:-}" ]]; then + params+=("--cluster-cidr=${CLUSTER_IP_RANGE}") + fi + if [[ -n "${CONCURRENT_SERVICE_SYNCS:-}" ]]; then + params+=("--concurrent-service-syncs=${CONCURRENT_SERVICE_SYNCS}") + fi + if [[ "${NETWORK_PROVIDER:-}" == "kubenet" ]]; then + params+=("--allocate-node-cidrs=true") + elif [[ -n "${ALLOCATE_NODE_CIDRS:-}" ]]; then + params+=("--allocate-node-cidrs=${ALLOCATE_NODE_CIDRS}") + fi + if [[ "${ENABLE_IP_ALIASES:-}" == 'true' ]]; then + params+=("--cidr-allocator-type=${NODE_IPAM_MODE}") + params+=("--configure-cloud-routes=false") + fi + if [[ -n "${FEATURE_GATES:-}" ]]; then + # remove non-GCP feature gates, since the CCM will early exit + # if given a feature gate it doesn't recognize + echo "Setting feature gates for cloud provider controller-manager from ${CCM_FEATURE_GATES}" + local CCM_FEATURE_GATES_FILTER + CCM_FEATURE_GATES_FILTER=$(echo "${CCM_FEATURE_GATES}" | sed "s/^/(/" | sed "s/,/=[^,]*|/g" | sed "s/$/=[^,]*)/") + echo "Computing safe feature gates for cloud provider controller-manager from ${FEATURE_GATES} and filter ${CCM_FEATURE_GATES_FILTER}" + local safe_feature_gates + safe_feature_gates=$(echo "${FEATURE_GATES}" | { grep -E -o "(${CCM_FEATURE_GATES_FILTER})" || true; } | tr "\n" "," | sed "s/,$//") + echo "Setting safe feature gates for cloud provider controller-manager with ${safe_feature_gates}" + if [[ -n "${safe_feature_gates:-}" ]]; then + params+=("--feature-gates=${safe_feature_gates}") + echo "Computing unsafe feature gates for cloud provider controller-manager from ${CCM_FEATURE_GATES_FILTER}" + local filtered_feature_gates + filtered_feature_gates=$(echo "${FEATURE_GATES}" | sed "s/,/\n/g" | { grep -E -v "(${CCM_FEATURE_GATES_FILTER})" || true; } | sed -z "s/\n/,/g;s/,$/\n/") + echo "Feature gates that did not pass through the GCP filter:" "${filtered_feature_gates}" + else + echo "None of the given feature gates (${FEATURE_GATES}) were found to be safe to pass to the CCM" + fi + fi + if [[ -n "${RUN_CONTROLLERS:-}" ]]; then + params+=("--controllers=${RUN_CONTROLLERS}") + fi + + echo "Converting manifest for cloud provider controller-manager" + local paramstring + paramstring="$(convert-manifest-params "${params[*]}")" + local container_env="" + if [[ -n "${ENABLE_CACHE_MUTATION_DETECTOR:-}" ]]; then + container_env="\"env\":[{\"name\": \"KUBE_CACHE_MUTATION_DETECTOR\", \"value\": \"${ENABLE_CACHE_MUTATION_DETECTOR}\"}]," + fi + + echo "Applying over-rides for manifest for cloud provider controller-manager" + local -r src_file="${KUBE_HOME}/kube-manifests/kubernetes/gci-trusty/cloud-controller-manager.manifest" + # Evaluate variables. + sed -i -e "s@{{pillar\['kube_docker_registry'\]}}@${DOCKER_REGISTRY}@g" "${src_file}" + sed -i -e "s@{{params}}@${paramstring}@g" "${src_file}" + sed -i -e "s@{{container_env}}@${container_env}@g" "${src_file}" + sed -i -e "s@{{cloud_config_mount}}@${CLOUD_CONFIG_MOUNT}@g" "${src_file}" + sed -i -e "s@{{cloud_config_volume}}@${CLOUD_CONFIG_VOLUME}@g" "${src_file}" + sed -i -e "s@{{additional_cloud_config_mount}}@@g" "${src_file}" + sed -i -e "s@{{additional_cloud_config_volume}}@@g" "${src_file}" + sed -i -e "s@{{pv_recycler_mount}}@${PV_RECYCLER_MOUNT}@g" "${src_file}" + sed -i -e "s@{{pv_recycler_volume}}@${PV_RECYCLER_VOLUME}@g" "${src_file}" + sed -i -e "s@{{flexvolume_hostpath_mount}}@${FLEXVOLUME_HOSTPATH_MOUNT}@g" "${src_file}" + sed -i -e "s@{{flexvolume_hostpath}}@${FLEXVOLUME_HOSTPATH_VOLUME}@g" "${src_file}" + sed -i -e "s@{{cpurequest}}@${CLOUD_CONTROLLER_MANAGER_CPU_REQUEST}@g" "${src_file}" + + if [[ -n "${CLOUD_CONTROLLER_MANAGER_RUNASUSER:-}" && -n "${CLOUD_CONTROLLER_MANAGER_RUNASGROUP:-}" ]]; then + #run-cloud-controller-manager-as-non-root + sed -i -e "s@{{runAsUser}}@\"runAsUser\": ${CLOUD_CONTROLLER_MANAGER_RUNASUSER},@g" "${src_file}" + sed -i -e "s@{{runAsGroup}}@\"runAsGroup\":${CLOUD_CONTROLLER_MANAGER_RUNASGROUP},@g" "${src_file}" + sed -i -e "s@{{supplementalGroups}}@\"supplementalGroups\": [ ${KUBE_PKI_READERS_GROUP} ],@g" "${src_file}" + else + sed -i -e "s@{{runAsUser}}@@g" "${src_file}" + sed -i -e "s@{{runAsGroup}}@@g" "${src_file}" + sed -i -e "s@{{supplementalGroups}}@@g" "${src_file}" + fi + + echo "Writing manifest for cloud provider controller-manager" + cp "${src_file}" /etc/kubernetes/manifests +} + # Starts kubernetes scheduler. # It prepares the log file, loads the docker image, calculates variables, sets them # in the manifest file, and then copies the manifest file to /etc/kubernetes/manifests. @@ -3329,6 +3440,7 @@ function main() { readonly KUBEDNS_AUTOSCALER="Deployment/kube-dns" # Resource requests of master components. + CLOUD_CONTROLLER_MANAGER_CPU_REQUEST="${KUBE_CONTROLLER_MANAGER_CPU_REQUEST:-50m}" KUBE_CONTROLLER_MANAGER_CPU_REQUEST="${KUBE_CONTROLLER_MANAGER_CPU_REQUEST:-200m}" KUBE_SCHEDULER_CPU_REQUEST="${KUBE_SCHEDULER_CPU_REQUEST:-75m}" @@ -3365,6 +3477,7 @@ function main() { log-start 'GenerateTokens' KUBE_CONTROLLER_MANAGER_TOKEN="$(secure_random 32)" + CLOUD_CONTROLLER_MANAGER_TOKEN="$(secure_random 32)" KUBE_SCHEDULER_TOKEN="$(secure_random 32)" KUBE_CLUSTER_AUTOSCALER_TOKEN="$(secure_random 32)" if [[ "${ENABLE_L7_LOADBALANCING:-}" == "glbc" ]]; then @@ -3459,6 +3572,10 @@ function main() { log-wrap 'StartKonnectivityServer' start-konnectivity-server fi log-wrap 'StartKubeControllerManager' start-kube-controller-manager + # (TODO/cloud-provider-gcp): Figure out how to inject + if [[ "${CLOUD_PROVIDER_FLAG:-gce}" == "external" ]]; then + log-wrap 'StartCloudControllerManager' start-cloud-controller-manager + fi log-wrap 'StartKubeScheduler' start-kube-scheduler log-wrap 'WaitTillApiserverReady' wait-till-apiserver-ready log-wrap 'StartKubeAddons' start-kube-addons diff --git a/cluster/gce/gci/configure.sh b/cluster/gce/gci/configure.sh index 9dbe34a6303..b320f73a211 100644 --- a/cluster/gce/gci/configure.sh +++ b/cluster/gce/gci/configure.sh @@ -572,7 +572,7 @@ kind: CredentialProviderConfig apiVersion: kubelet.config.k8s.io/v1beta1 providers: - name: auth-provider-gcp - apiVersion: credentialprovider.kubelet.k8s.io/v1beta1 + apiVersion: credentialprovider.kubelet.k8s.io/v1alpha1 matchImages: - "container.cloud.google.com" - "gcr.io" diff --git a/cluster/gce/manifests/cloud-controller-manager.manifest b/cluster/gce/manifests/cloud-controller-manager.manifest new file mode 100644 index 00000000000..dc885a810ad --- /dev/null +++ b/cluster/gce/manifests/cloud-controller-manager.manifest @@ -0,0 +1,113 @@ +{ +"apiVersion": "v1", +"kind": "Pod", +"metadata": { + "name":"cloud-controller-manager", + "namespace": "kube-system", + "labels": { + "tier": "control-plane", + "component": "cloud-controller-manager" + } +}, +"spec":{ +"securityContext": { + {{runAsUser}} + {{runAsGroup}} + {{supplementalGroups}} + "seccompProfile": { + "type": "RuntimeDefault" + } +}, +"priorityClass": "system-node-critical", +"hostNetwork": true, +"containers":[ + { + "name": "cloud-controller-manager", + "image": "gcr.io/k8s-staging-cloud-provider-gcp/cloud-controller-manager:v1.25.2-alpha_ae91c1fc0c443c464a4c878ffa2a4544483c6d1f", + "resources": { + "requests": { + "cpu": "{{cpurequest}}" + } + }, + "command": ["/cloud-controller-manager"], + "args": [ + "--log-file=/var/log/cloud-controller-manager.log", + "--logtostderr=false", + {{params}} + ], + {{container_env}} + "livenessProbe": { + "httpGet": { + "host": "127.0.0.1", + "port": 10258, + "scheme": "HTTPS", + "path": "/healthz" + }, + "initialDelaySeconds": 15, + "timeoutSeconds": 15 + }, + "volumeMounts": [ + {{cloud_config_mount}} + {{additional_cloud_config_mount}} + {{pv_recycler_mount}} + { "name": "srvkube", + "mountPath": "/etc/srv/kubernetes", + "readOnly": true}, + {{flexvolume_hostpath_mount}} + { "name": "logfile", + "mountPath": "/var/log/cloud-controller-manager.log", + "readOnly": false}, + { "name": "etcssl", + "mountPath": "/etc/ssl", + "readOnly": true}, + { "name": "usrsharecacerts", + "mountPath": "/usr/share/ca-certificates", + "readOnly": true}, + { "name": "varssl", + "mountPath": "/var/ssl", + "readOnly": true}, + { "name": "etcopenssl", + "mountPath": "/etc/openssl", + "readOnly": true}, + { "name": "etcpki", + "mountPath": "/etc/pki", + "readOnly": true} + ] + } +], +"volumes":[ + {{cloud_config_volume}} + {{additional_cloud_config_volume}} + {{pv_recycler_volume}} + { "name": "srvkube", + "hostPath": { + "path": "/etc/srv/kubernetes"} + }, + {{flexvolume_hostpath}} + { "name": "logfile", + "hostPath": { + "path": "/var/log/cloud-controller-manager.log", + "type": "FileOrCreate"} + }, + { "name": "etcssl", + "hostPath": { + "path": "/etc/ssl"} + }, + { "name": "usrsharecacerts", + "hostPath": { + "path": "/usr/share/ca-certificates"} + }, + { "name": "varssl", + "hostPath": { + "path": "/var/ssl"} + }, + { "name": "etcopenssl", + "hostPath": { + "path": "/etc/openssl"} + }, + { "name": "etcpki", + "hostPath": { + "path": "/etc/pki"} + } +] +}} diff --git a/cluster/gce/util.sh b/cluster/gce/util.sh index f86a4ca3f82..7df28267e03 100755 --- a/cluster/gce/util.sh +++ b/cluster/gce/util.sh @@ -1316,6 +1316,7 @@ ETCD_PEER_KEY: $(yaml-quote "${ETCD_PEER_KEY_BASE64:-}") ETCD_PEER_CERT: $(yaml-quote "${ETCD_PEER_CERT_BASE64:-}") SERVICEACCOUNT_ISSUER: $(yaml-quote "${SERVICEACCOUNT_ISSUER:-}") KUBECTL_PRUNE_WHITELIST_OVERRIDE: $(yaml-quote "${KUBECTL_PRUNE_WHITELIST_OVERRIDE:-}") +CCM_FEATURE_GATES: $(yaml-quote "${CCM_FEATURE_GATES:-}") KUBE_SCHEDULER_RUNASUSER: 2001 KUBE_SCHEDULER_RUNASGROUP: 2001 KUBE_ADDON_MANAGER_RUNASUSER: 2002