From 75e602ca28367fe0a4aa20f36bf1a31c698d0d47 Mon Sep 17 00:00:00 2001 From: Shyam JVS Date: Mon, 27 Feb 2017 14:35:12 -0800 Subject: [PATCH] Convert hollow-node manifest to yaml and add init container for setting inotify limit --- .../resources/hollow-node_template.json | 198 ------------------ .../resources/hollow-node_template.yaml | 126 +++++++++++ test/kubemark/start-kubemark.sh | 14 +- 3 files changed, 133 insertions(+), 205 deletions(-) delete mode 100644 test/kubemark/resources/hollow-node_template.json create mode 100644 test/kubemark/resources/hollow-node_template.yaml diff --git a/test/kubemark/resources/hollow-node_template.json b/test/kubemark/resources/hollow-node_template.json deleted file mode 100644 index 974fe110d8c..00000000000 --- a/test/kubemark/resources/hollow-node_template.json +++ /dev/null @@ -1,198 +0,0 @@ -{ - "kind": "ReplicationController", - "apiVersion": "v1", - "metadata": { - "name": "hollow-node", - "labels": { - "name": "hollow-node" - } - }, - "spec": { - "replicas": {{numreplicas}}, - "selector": { - "name": "hollow-node" - }, - "template": { - "metadata": { - "labels": { - "name": "hollow-node" - } - }, - "spec": { - "volumes": [ - { - "name": "kubeconfig-volume", - "secret": { - "secretName": "kubeconfig" - } - }, - { - "name": "kernelmonitorconfig-volume", - "configMap": { - "name": "node-configmap" - } - }, - { - "name": "logs-volume", - "hostPath": { - "path": "/var/log" - } - }, - { - "name": "no-serviceaccount-access-to-real-master", - "emptyDir": {} - } - ], - "containers": [ - { - "name": "hollow-kubelet", - "image": "{{registry}}/{{project}}/kubemark:latest", - "ports": [ - {"containerPort": 4194}, - {"containerPort": 10250}, - {"containerPort": 10255} - ], - "env": [ - { - "name": "CONTENT_TYPE", - "valueFrom": { - "configMapKeyRef": { - "name": "node-configmap", - "key": "content.type" - } - } - }, - { - "name": "NODE_NAME", - "valueFrom": { - "fieldRef": { - "fieldPath": "metadata.name" - } - } - } - ], - "command": [ - "/bin/sh", - "-c", - "./kubemark.sh --morph=kubelet --name=$(NODE_NAME) --kubeconfig=/kubeconfig/kubelet.kubeconfig $(CONTENT_TYPE) --v=2 1>>/var/log/kubelet-$(NODE_NAME).log 2>&1" - ], - "volumeMounts": [ - { - "name": "kubeconfig-volume", - "mountPath": "/kubeconfig", - "readOnly": true - }, - { - "name": "logs-volume", - "mountPath": "/var/log" - } - ], - "resources": { - "requests": { - "cpu": "50m", - "memory": "100M" - } - }, - "securityContext": { - "privileged": true - } - - }, - { - "name": "hollow-proxy", - "image": "{{registry}}/{{project}}/kubemark:latest", - "env": [ - { - "name": "CONTENT_TYPE", - "valueFrom": { - "configMapKeyRef": { - "name": "node-configmap", - "key": "content.type" - } - } - }, - { - "name": "NODE_NAME", - "valueFrom": { - "fieldRef": { - "fieldPath": "metadata.name" - } - } - } - ], - "command": [ - "/bin/sh", - "-c", - "./kubemark.sh --morph=proxy --name=$(NODE_NAME) --kubeconfig=/kubeconfig/kubeproxy.kubeconfig $(CONTENT_TYPE) --v=2 1>>/var/log/kubeproxy-$(NODE_NAME).log 2>&1" - ], - "volumeMounts": [ - { - "name": "kubeconfig-volume", - "mountPath": "/kubeconfig", - "readOnly": true - }, - { - "name": "logs-volume", - "mountPath": "/var/log" - } - ], - "resources": { - "requests": { - "cpu": "{{HOLLOW_PROXY_CPU}}m", - "memory": "{{HOLLOW_PROXY_MEM}}Ki" - } - } - }, - { - "name": "hollow-node-problem-detector", - "image": "gcr.io/google_containers/node-problem-detector:v0.3.0-alpha.0", - "env": [ - { - "name": "NODE_NAME", - "valueFrom": { - "fieldRef": { - "fieldPath": "metadata.name" - } - } - } - ], - "command": [ - "/bin/sh", - "-c", - "/node-problem-detector --kernel-monitor=/config/kernel.monitor --apiserver-override=\"https://{{master_ip}}:443?inClusterConfig=false&auth=/kubeconfig/npd.kubeconfig\" --alsologtostderr 1>>/var/log/npd-$(NODE_NAME).log 2>&1" - ], - "volumeMounts": [ - { - "name": "kubeconfig-volume", - "mountPath": "/kubeconfig", - "readOnly": true - }, - { - "name": "kernelmonitorconfig-volume", - "mountPath": "/config", - "readOnly": true - }, - { - "name": "no-serviceaccount-access-to-real-master", - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "readOnly": true - }, - { - "name": "logs-volume", - "mountPath": "/var/log" - } - ], - "resources": { - "requests": { - "cpu": "20m", - "memory": "20Mi" - } - }, - "securityContext": { - "privileged": true - } - }] - } - } - } -} diff --git a/test/kubemark/resources/hollow-node_template.yaml b/test/kubemark/resources/hollow-node_template.yaml new file mode 100644 index 00000000000..fc0dd5146f7 --- /dev/null +++ b/test/kubemark/resources/hollow-node_template.yaml @@ -0,0 +1,126 @@ +apiVersion: v1 +kind: ReplicationController +metadata: + name: hollow-node + labels: + name: hollow-node +spec: + replicas: {{numreplicas}} + selector: + name: hollow-node + template: + metadata: + labels: + name: hollow-node + annotations: + pod.beta.kubernetes.io/init-containers: '[ + { + "name": "init-inotify-limit", + "image": "busybox", + "command": ["sysctl", "-w", "fs.inotify.max_user_instances=200"], + "securityContext": { + "privileged": true + } + } + ]' + spec: + volumes: + - name: kubeconfig-volume + secret: + secretName: kubeconfig + - name: kernelmonitorconfig-volume + configMap: + name: node-configmap + - name: logs-volume + hostPath: + path: /var/log + - name: no-serviceaccount-access-to-real-master + emptyDir: {} + containers: + - name: hollow-kubelet + image: {{registry}}/{{project}}/kubemark:latest + ports: + - containerPort: 4194 + - containerPort: 10250 + - containerPort: 10255 + env: + - name: CONTENT_TYPE + valueFrom: + configMapKeyRef: + name: node-configmap + key: content.type + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + command: + - /bin/sh + - -c + - ./kubemark.sh --morph=kubelet --name=$(NODE_NAME) --kubeconfig=/kubeconfig/kubelet.kubeconfig $(CONTENT_TYPE) --v=2 1>>/var/log/kubelet-$(NODE_NAME).log 2>&1 + volumeMounts: + - name: kubeconfig-volume + mountPath: /kubeconfig + readOnly: true + - name: logs-volume + mountPath: /var/log + resources: + requests: + cpu: 50m + memory: 100M + securityContext: + privileged: true + - name: hollow-proxy + image: {{registry}}/{{project}}/kubemark:latest + env: + - name: CONTENT_TYPE + valueFrom: + configMapKeyRef: + name: node-configmap + key: content.type + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + command: + - /bin/sh + - -c + - ./kubemark.sh --morph=proxy --name=$(NODE_NAME) --kubeconfig=/kubeconfig/kubeproxy.kubeconfig $(CONTENT_TYPE) --v=2 1>>/var/log/kubeproxy-$(NODE_NAME).log 2>&1 + volumeMounts: + - name: kubeconfig-volume + mountPath: /kubeconfig + readOnly: true + - name: logs-volume + mountPath: /var/log + resources: + requests: + cpu: {{HOLLOW_PROXY_CPU}}m + memory: {{HOLLOW_PROXY_MEM}}Ki + - name: hollow-node-problem-detector + image: gcr.io/google_containers/node-problem-detector:v0.3.0-alpha.0 + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + command: + - /bin/sh + - -c + - /node-problem-detector --kernel-monitor=/config/kernel.monitor --apiserver-override="https://{{master_ip}}:443?inClusterConfig=false&auth=/kubeconfig/npd.kubeconfig" --alsologtostderr 1>>/var/log/npd-$(NODE_NAME).log 2>&1 + volumeMounts: + - name: kubeconfig-volume + mountPath: /kubeconfig + readOnly: true + - name: kernelmonitorconfig-volume + mountPath: /config + readOnly: true + - name: no-serviceaccount-access-to-real-master + mountPath: /var/run/secrets/kubernetes.io/serviceaccount + readOnly: true + - name: logs-volume + mountPath: /var/log + resources: + requests: + cpu: 20m + memory: 20Mi + securityContext: + privileged: true diff --git a/test/kubemark/start-kubemark.sh b/test/kubemark/start-kubemark.sh index f46900cef11..90a32623c62 100755 --- a/test/kubemark/start-kubemark.sh +++ b/test/kubemark/start-kubemark.sh @@ -308,19 +308,19 @@ current-context: kubemark-context") "${KUBECTL}" create -f "${RESOURCE_DIRECTORY}/addons" --namespace="kubemark" # Create the replication controller for hollow-nodes. - sed "s/{{numreplicas}}/${NUM_NODES:-10}/g" "${RESOURCE_DIRECTORY}/hollow-node_template.json" > "${RESOURCE_DIRECTORY}/hollow-node.json" + sed "s/{{numreplicas}}/${NUM_NODES:-10}/g" "${RESOURCE_DIRECTORY}/hollow-node_template.yaml" > "${RESOURCE_DIRECTORY}/hollow-node.yaml" proxy_cpu=20 if [ "${NUM_NODES:-10}" -gt 1000 ]; then proxy_cpu=50 fi proxy_mem_per_node=100 proxy_mem=$((100 * 1024 + ${proxy_mem_per_node}*${NUM_NODES:-10})) - sed -i'' -e "s/{{HOLLOW_PROXY_CPU}}/${proxy_cpu}/g" "${RESOURCE_DIRECTORY}/hollow-node.json" - sed -i'' -e "s/{{HOLLOW_PROXY_MEM}}/${proxy_mem}/g" "${RESOURCE_DIRECTORY}/hollow-node.json" - sed -i'' -e "s/{{registry}}/${CONTAINER_REGISTRY}/g" "${RESOURCE_DIRECTORY}/hollow-node.json" - sed -i'' -e "s/{{project}}/${PROJECT}/g" "${RESOURCE_DIRECTORY}/hollow-node.json" - sed -i'' -e "s/{{master_ip}}/${MASTER_IP}/g" "${RESOURCE_DIRECTORY}/hollow-node.json" - "${KUBECTL}" create -f "${RESOURCE_DIRECTORY}/hollow-node.json" --namespace="kubemark" + sed -i'' -e "s/{{HOLLOW_PROXY_CPU}}/${proxy_cpu}/g" "${RESOURCE_DIRECTORY}/hollow-node.yaml" + sed -i'' -e "s/{{HOLLOW_PROXY_MEM}}/${proxy_mem}/g" "${RESOURCE_DIRECTORY}/hollow-node.yaml" + sed -i'' -e "s/{{registry}}/${CONTAINER_REGISTRY}/g" "${RESOURCE_DIRECTORY}/hollow-node.yaml" + sed -i'' -e "s/{{project}}/${PROJECT}/g" "${RESOURCE_DIRECTORY}/hollow-node.yaml" + sed -i'' -e "s/{{master_ip}}/${MASTER_IP}/g" "${RESOURCE_DIRECTORY}/hollow-node.yaml" + "${KUBECTL}" create -f "${RESOURCE_DIRECTORY}/hollow-node.yaml" --namespace="kubemark" echo "Created secrets, configMaps, replication-controllers required for hollow-nodes." }