From 3de7e5ab407107648689e5738da745aa27c87fae Mon Sep 17 00:00:00 2001 From: Rohit Agarwal Date: Fri, 27 Oct 2017 12:50:09 -0700 Subject: [PATCH 1/4] Remove redundant comment and improve documentation. The comment is also present in lines 143-145 where it makes more sense. --- cluster/addons/addon-manager/README.md | 11 ++++++----- cluster/addons/addon-manager/kube-addons.sh | 3 --- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/cluster/addons/addon-manager/README.md b/cluster/addons/addon-manager/README.md index b9da7039ba4..e9ae53458c9 100644 --- a/cluster/addons/addon-manager/README.md +++ b/cluster/addons/addon-manager/README.md @@ -1,26 +1,27 @@ ### Addon-manager -addon-manager manages two classes of addons with given template files. +addon-manager manages two classes of addons with given template files in +`$ADDON_PATH` (default `/etc/kubernetes/addons/`). - Addons with label `addonmanager.kubernetes.io/mode=Reconcile` will be periodically reconciled. Direct manipulation to these addons through apiserver is discouraged because addon-manager will bring them back to the original state. In particular: - Addon will be re-created if it is deleted. - Addon will be reconfigured to the state given by the supplied fields in the template file periodically. - - Addon will be deleted when its manifest file is deleted. + - Addon will be deleted when its manifest file is deleted from the `$ADDON_PATH`. - Addons with label `addonmanager.kubernetes.io/mode=EnsureExists` will be checked for existence only. Users can edit these addons as they want. In particular: - Addon will only be created/re-created with the given template file when there is no instance of the resource with that name. - - Addon will not be deleted when the manifest file is deleted. + - Addon will not be deleted when the manifest file is deleted from the `$ADDON_PATH`. Notes: - Label `kubernetes.io/cluster-service=true` is deprecated (only for Addon Manager). In future release (after one year), Addon Manager may not respect it anymore. Addons have this label but without `addonmanager.kubernetes.io/mode=EnsureExists` will be treated as "reconcile class addons" for now. -- Resources under $ADDON_PATH (default `/etc/kubernetes/addons/`) needs to have either one -of these two labels. Meanwhile namespaced resources need to be in `kube-system` namespace. +- Resources under `$ADDON_PATH` need to have either one of these two labels. +Meanwhile namespaced resources need to be in `kube-system` namespace. Otherwise it will be omitted. - The above label and namespace rule does not stand for `/opt/namespace.yaml` and resources under `/etc/kubernetes/admission-controls/`. addon-manager will attempt to diff --git a/cluster/addons/addon-manager/kube-addons.sh b/cluster/addons/addon-manager/kube-addons.sh index 84106852025..629ebe66c8e 100755 --- a/cluster/addons/addon-manager/kube-addons.sh +++ b/cluster/addons/addon-manager/kube-addons.sh @@ -26,9 +26,6 @@ # 3. Kubectl prints the output to stderr (the output should be captured and then # logged) -# The business logic for whether a given object should be created -# was already enforced by salt, and /etc/kubernetes/addons is the -# managed result is of that. Start everything below that directory. KUBECTL=${KUBECTL_BIN:-/usr/local/bin/kubectl} KUBECTL_OPTS=${KUBECTL_OPTS:-} From e196b2ebfbcf61d6452e658dc830b159615edac1 Mon Sep 17 00:00:00 2001 From: Rohit Agarwal Date: Mon, 30 Oct 2017 15:45:06 -0700 Subject: [PATCH 2/4] Enable DevicePlugins feature on GCE clusters with accelerators attached. Instead of the old Accelerators feature that added alpha.kubernetes.io/nvidia-gpu resource, use the new DevicePlugins feature that adds vendor specific resources. (In case of nvidia it will add nvidia.com/gpu resource.) --- cluster/gce/config-default.sh | 2 +- cluster/gce/config-test.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cluster/gce/config-default.sh b/cluster/gce/config-default.sh index a88885ecb32..460b09c0069 100755 --- a/cluster/gce/config-default.sh +++ b/cluster/gce/config-default.sh @@ -182,7 +182,7 @@ RUNTIME_CONFIG="${KUBE_RUNTIME_CONFIG:-}" FEATURE_GATES="${KUBE_FEATURE_GATES:-ExperimentalCriticalPodAnnotation=true}" if [[ ! -z "${NODE_ACCELERATORS}" ]]; then - FEATURE_GATES="${FEATURE_GATES},Accelerators=true" + FEATURE_GATES="${FEATURE_GATES},DevicePlugins=true" fi # Optional: Install cluster DNS. diff --git a/cluster/gce/config-test.sh b/cluster/gce/config-test.sh index 426ac4f3a92..94b36e69921 100755 --- a/cluster/gce/config-test.sh +++ b/cluster/gce/config-test.sh @@ -109,7 +109,7 @@ RUNTIME_CONFIG="${KUBE_RUNTIME_CONFIG:-}" FEATURE_GATES="${KUBE_FEATURE_GATES:-ExperimentalCriticalPodAnnotation=true}" if [[ ! -z "${NODE_ACCELERATORS}" ]]; then - FEATURE_GATES="${FEATURE_GATES},Accelerators=true" + FEATURE_GATES="${FEATURE_GATES},DevicePlugins=true" fi TERMINATED_POD_GC_THRESHOLD=${TERMINATED_POD_GC_THRESHOLD:-100} From 9c7baf94dd72c4b84f423625a0ae138df9d37317 Mon Sep 17 00:00:00 2001 From: Rohit Agarwal Date: Mon, 30 Oct 2017 15:48:39 -0700 Subject: [PATCH 3/4] Add node label to GCE nodes with accelerators attached. This node label is the same as what GKE attaches to node pools with accelerators attached. This will help us target accelerator specific daemonsets etc. to these nodes. --- cluster/gce/config-default.sh | 3 +++ cluster/gce/config-test.sh | 11 +++++++---- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/cluster/gce/config-default.sh b/cluster/gce/config-default.sh index 460b09c0069..90433158b19 100755 --- a/cluster/gce/config-default.sh +++ b/cluster/gce/config-default.sh @@ -183,6 +183,9 @@ FEATURE_GATES="${KUBE_FEATURE_GATES:-ExperimentalCriticalPodAnnotation=true}" if [[ ! -z "${NODE_ACCELERATORS}" ]]; then FEATURE_GATES="${FEATURE_GATES},DevicePlugins=true" + if [[ "${NODE_ACCELERATORS}" =~ .*type=([a-zA-Z0-9-]+).* ]]; then + NODE_LABELS="${NODE_LABELS},cloud.google.com/gke-accelerator=${BASH_REMATCH[1]}" + fi fi # Optional: Install cluster DNS. diff --git a/cluster/gce/config-test.sh b/cluster/gce/config-test.sh index 94b36e69921..7c6c230537a 100755 --- a/cluster/gce/config-test.sh +++ b/cluster/gce/config-test.sh @@ -108,10 +108,6 @@ RUNTIME_CONFIG="${KUBE_RUNTIME_CONFIG:-}" # Optional: set feature gates FEATURE_GATES="${KUBE_FEATURE_GATES:-ExperimentalCriticalPodAnnotation=true}" -if [[ ! -z "${NODE_ACCELERATORS}" ]]; then - FEATURE_GATES="${FEATURE_GATES},DevicePlugins=true" -fi - TERMINATED_POD_GC_THRESHOLD=${TERMINATED_POD_GC_THRESHOLD:-100} # Extra docker options for nodes. @@ -228,6 +224,13 @@ if [[ ${KUBE_ENABLE_INSECURE_REGISTRY:-false} == "true" ]]; then EXTRA_DOCKER_OPTS="${EXTRA_DOCKER_OPTS} --insecure-registry 10.0.0.0/8" fi +if [[ ! -z "${NODE_ACCELERATORS}" ]]; then + FEATURE_GATES="${FEATURE_GATES},DevicePlugins=true" + if [[ "${NODE_ACCELERATORS}" =~ .*type=([a-zA-Z0-9-]+).* ]]; then + NODE_LABELS="${NODE_LABELS},cloud.google.com/gke-accelerator=${BASH_REMATCH[1]}" + fi +fi + # Optional: Install cluster DNS. ENABLE_CLUSTER_DNS="${KUBE_ENABLE_CLUSTER_DNS:-true}" DNS_SERVER_IP="10.0.0.10" From cf292754ba423aa6782564ea83fe48cc1ed677d4 Mon Sep 17 00:00:00 2001 From: Rohit Agarwal Date: Mon, 30 Oct 2017 15:50:08 -0700 Subject: [PATCH 4/4] Run nvidia-gpu device-plugin daemonset as an addon on GCE nodes that have nvidia GPUs attached. --- .../device-plugins/nvidia-gpu/daemonset.yaml | 45 +++++++++++++++++++ cluster/common.sh | 5 +++ cluster/gce/gci/configure-helper.sh | 3 ++ 3 files changed, 53 insertions(+) create mode 100644 cluster/addons/device-plugins/nvidia-gpu/daemonset.yaml diff --git a/cluster/addons/device-plugins/nvidia-gpu/daemonset.yaml b/cluster/addons/device-plugins/nvidia-gpu/daemonset.yaml new file mode 100644 index 00000000000..a5eaf1da76d --- /dev/null +++ b/cluster/addons/device-plugins/nvidia-gpu/daemonset.yaml @@ -0,0 +1,45 @@ +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: nvidia-gpu-device-plugin + namespace: kube-system + labels: + k8s-app: nvidia-gpu-device-plugin + addonmanager.kubernetes.io/mode: Reconcile +spec: + template: + metadata: + labels: + k8s-app: nvidia-gpu-device-plugin + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: cloud.google.com/gke-accelerator + operator: Exists + hostNetwork: true + hostPID: true + volumes: + - name: device-plugin + hostPath: + path: /var/lib/kubelet/device-plugins + - name: dev + hostPath: + path: /dev + containers: + - image: "gcr.io/google-containers/nvidia-gpu-device-plugin@sha256:943a62949cd80c26e7371d4e123dac61b4cc7281390721aaa95f265171094842" + command: ["/usr/bin/nvidia-gpu-device-plugin", "-logtostderr"] + name: nvidia-gpu-device-plugin + resources: + requests: + cpu: 10m + memory: 10Mi + securityContext: + privileged: true + volumeMounts: + - name: device-plugin + mountPath: /device-plugin + - name: dev + mountPath: /dev diff --git a/cluster/common.sh b/cluster/common.sh index 2b11917d776..dc0107f0271 100755 --- a/cluster/common.sh +++ b/cluster/common.sh @@ -873,6 +873,11 @@ EOF if [ -n "${CLUSTER_SIGNING_DURATION:-}" ]; then cat >>$file <>$file <