mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-22 11:21:47 +00:00
Merge pull request #42201 from shyamjvs/inotify-limit
Automatic merge from submit-queue (batch tested with PRs 42316, 41618, 42201, 42113, 42191) [Kubemark] Add init container in hollow node for setting inotify limit of node to 200 Fixes #41713 Along with adding the init container, I also changed the manifest to a yaml as otherwise the entire init container annotation would have to be in a single line (with escaped characters), as json doesn't allow multi-line strings. cc @kubernetes/sig-scalability-misc @wojtek-t @gmarek @Random-Liu
This commit is contained in:
Kubernetes Submit Queue
GitHub
commit
32d59cbb2f
@ -1,198 +0,0 @@
|
||||
{
|
||||
"kind": "ReplicationController",
|
||||
"apiVersion": "v1",
|
||||
"metadata": {
|
||||
"name": "hollow-node",
|
||||
"labels": {
|
||||
"name": "hollow-node"
|
||||
}
|
||||
},
|
||||
"spec": {
|
||||
"replicas": {{numreplicas}},
|
||||
"selector": {
|
||||
"name": "hollow-node"
|
||||
},
|
||||
"template": {
|
||||
"metadata": {
|
||||
"labels": {
|
||||
"name": "hollow-node"
|
||||
}
|
||||
},
|
||||
"spec": {
|
||||
"volumes": [
|
||||
{
|
||||
"name": "kubeconfig-volume",
|
||||
"secret": {
|
||||
"secretName": "kubeconfig"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "kernelmonitorconfig-volume",
|
||||
"configMap": {
|
||||
"name": "node-configmap"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "logs-volume",
|
||||
"hostPath": {
|
||||
"path": "/var/log"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "no-serviceaccount-access-to-real-master",
|
||||
"emptyDir": {}
|
||||
}
|
||||
],
|
||||
"containers": [
|
||||
{
|
||||
"name": "hollow-kubelet",
|
||||
"image": "{{registry}}/{{project}}/kubemark:latest",
|
||||
"ports": [
|
||||
{"containerPort": 4194},
|
||||
{"containerPort": 10250},
|
||||
{"containerPort": 10255}
|
||||
],
|
||||
"env": [
|
||||
{
|
||||
"name": "CONTENT_TYPE",
|
||||
"valueFrom": {
|
||||
"configMapKeyRef": {
|
||||
"name": "node-configmap",
|
||||
"key": "content.type"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "NODE_NAME",
|
||||
"valueFrom": {
|
||||
"fieldRef": {
|
||||
"fieldPath": "metadata.name"
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"command": [
|
||||
"/bin/sh",
|
||||
"-c",
|
||||
"./kubemark.sh --morph=kubelet --name=$(NODE_NAME) --kubeconfig=/kubeconfig/kubelet.kubeconfig $(CONTENT_TYPE) --v=2 1>>/var/log/kubelet-$(NODE_NAME).log 2>&1"
|
||||
],
|
||||
"volumeMounts": [
|
||||
{
|
||||
"name": "kubeconfig-volume",
|
||||
"mountPath": "/kubeconfig",
|
||||
"readOnly": true
|
||||
},
|
||||
{
|
||||
"name": "logs-volume",
|
||||
"mountPath": "/var/log"
|
||||
}
|
||||
],
|
||||
"resources": {
|
||||
"requests": {
|
||||
"cpu": "50m",
|
||||
"memory": "100M"
|
||||
}
|
||||
},
|
||||
"securityContext": {
|
||||
"privileged": true
|
||||
}
|
||||
|
||||
},
|
||||
{
|
||||
"name": "hollow-proxy",
|
||||
"image": "{{registry}}/{{project}}/kubemark:latest",
|
||||
"env": [
|
||||
{
|
||||
"name": "CONTENT_TYPE",
|
||||
"valueFrom": {
|
||||
"configMapKeyRef": {
|
||||
"name": "node-configmap",
|
||||
"key": "content.type"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "NODE_NAME",
|
||||
"valueFrom": {
|
||||
"fieldRef": {
|
||||
"fieldPath": "metadata.name"
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"command": [
|
||||
"/bin/sh",
|
||||
"-c",
|
||||
"./kubemark.sh --morph=proxy --name=$(NODE_NAME) --kubeconfig=/kubeconfig/kubeproxy.kubeconfig $(CONTENT_TYPE) --v=2 1>>/var/log/kubeproxy-$(NODE_NAME).log 2>&1"
|
||||
],
|
||||
"volumeMounts": [
|
||||
{
|
||||
"name": "kubeconfig-volume",
|
||||
"mountPath": "/kubeconfig",
|
||||
"readOnly": true
|
||||
},
|
||||
{
|
||||
"name": "logs-volume",
|
||||
"mountPath": "/var/log"
|
||||
}
|
||||
],
|
||||
"resources": {
|
||||
"requests": {
|
||||
"cpu": "{{HOLLOW_PROXY_CPU}}m",
|
||||
"memory": "{{HOLLOW_PROXY_MEM}}Ki"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "hollow-node-problem-detector",
|
||||
"image": "gcr.io/google_containers/node-problem-detector:v0.3.0-alpha.0",
|
||||
"env": [
|
||||
{
|
||||
"name": "NODE_NAME",
|
||||
"valueFrom": {
|
||||
"fieldRef": {
|
||||
"fieldPath": "metadata.name"
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"command": [
|
||||
"/bin/sh",
|
||||
"-c",
|
||||
"/node-problem-detector --kernel-monitor=/config/kernel.monitor --apiserver-override=\"https://{{master_ip}}:443?inClusterConfig=false&auth=/kubeconfig/npd.kubeconfig\" --alsologtostderr 1>>/var/log/npd-$(NODE_NAME).log 2>&1"
|
||||
],
|
||||
"volumeMounts": [
|
||||
{
|
||||
"name": "kubeconfig-volume",
|
||||
"mountPath": "/kubeconfig",
|
||||
"readOnly": true
|
||||
},
|
||||
{
|
||||
"name": "kernelmonitorconfig-volume",
|
||||
"mountPath": "/config",
|
||||
"readOnly": true
|
||||
},
|
||||
{
|
||||
"name": "no-serviceaccount-access-to-real-master",
|
||||
"mountPath": "/var/run/secrets/kubernetes.io/serviceaccount",
|
||||
"readOnly": true
|
||||
},
|
||||
{
|
||||
"name": "logs-volume",
|
||||
"mountPath": "/var/log"
|
||||
}
|
||||
],
|
||||
"resources": {
|
||||
"requests": {
|
||||
"cpu": "20m",
|
||||
"memory": "20Mi"
|
||||
}
|
||||
},
|
||||
"securityContext": {
|
||||
"privileged": true
|
||||
}
|
||||
}]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
126
test/kubemark/resources/hollow-node_template.yaml
Normal file
126
test/kubemark/resources/hollow-node_template.yaml
Normal file
@ -0,0 +1,126 @@
|
||||
apiVersion: v1
|
||||
kind: ReplicationController
|
||||
metadata:
|
||||
name: hollow-node
|
||||
labels:
|
||||
name: hollow-node
|
||||
spec:
|
||||
replicas: {{numreplicas}}
|
||||
selector:
|
||||
name: hollow-node
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
name: hollow-node
|
||||
annotations:
|
||||
pod.beta.kubernetes.io/init-containers: '[
|
||||
{
|
||||
"name": "init-inotify-limit",
|
||||
"image": "busybox",
|
||||
"command": ["sysctl", "-w", "fs.inotify.max_user_instances=200"],
|
||||
"securityContext": {
|
||||
"privileged": true
|
||||
}
|
||||
}
|
||||
]'
|
||||
spec:
|
||||
volumes:
|
||||
- name: kubeconfig-volume
|
||||
secret:
|
||||
secretName: kubeconfig
|
||||
- name: kernelmonitorconfig-volume
|
||||
configMap:
|
||||
name: node-configmap
|
||||
- name: logs-volume
|
||||
hostPath:
|
||||
path: /var/log
|
||||
- name: no-serviceaccount-access-to-real-master
|
||||
emptyDir: {}
|
||||
containers:
|
||||
- name: hollow-kubelet
|
||||
image: {{registry}}/{{project}}/kubemark:latest
|
||||
ports:
|
||||
- containerPort: 4194
|
||||
- containerPort: 10250
|
||||
- containerPort: 10255
|
||||
env:
|
||||
- name: CONTENT_TYPE
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
name: node-configmap
|
||||
key: content.type
|
||||
- name: NODE_NAME
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: metadata.name
|
||||
command:
|
||||
- /bin/sh
|
||||
- -c
|
||||
- ./kubemark.sh --morph=kubelet --name=$(NODE_NAME) --kubeconfig=/kubeconfig/kubelet.kubeconfig $(CONTENT_TYPE) --v=2 1>>/var/log/kubelet-$(NODE_NAME).log 2>&1
|
||||
volumeMounts:
|
||||
- name: kubeconfig-volume
|
||||
mountPath: /kubeconfig
|
||||
readOnly: true
|
||||
- name: logs-volume
|
||||
mountPath: /var/log
|
||||
resources:
|
||||
requests:
|
||||
cpu: 50m
|
||||
memory: 100M
|
||||
securityContext:
|
||||
privileged: true
|
||||
- name: hollow-proxy
|
||||
image: {{registry}}/{{project}}/kubemark:latest
|
||||
env:
|
||||
- name: CONTENT_TYPE
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
name: node-configmap
|
||||
key: content.type
|
||||
- name: NODE_NAME
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: metadata.name
|
||||
command:
|
||||
- /bin/sh
|
||||
- -c
|
||||
- ./kubemark.sh --morph=proxy --name=$(NODE_NAME) --kubeconfig=/kubeconfig/kubeproxy.kubeconfig $(CONTENT_TYPE) --v=2 1>>/var/log/kubeproxy-$(NODE_NAME).log 2>&1
|
||||
volumeMounts:
|
||||
- name: kubeconfig-volume
|
||||
mountPath: /kubeconfig
|
||||
readOnly: true
|
||||
- name: logs-volume
|
||||
mountPath: /var/log
|
||||
resources:
|
||||
requests:
|
||||
cpu: {{HOLLOW_PROXY_CPU}}m
|
||||
memory: {{HOLLOW_PROXY_MEM}}Ki
|
||||
- name: hollow-node-problem-detector
|
||||
image: gcr.io/google_containers/node-problem-detector:v0.3.0-alpha.0
|
||||
env:
|
||||
- name: NODE_NAME
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: metadata.name
|
||||
command:
|
||||
- /bin/sh
|
||||
- -c
|
||||
- /node-problem-detector --kernel-monitor=/config/kernel.monitor --apiserver-override="https://{{master_ip}}:443?inClusterConfig=false&auth=/kubeconfig/npd.kubeconfig" --alsologtostderr 1>>/var/log/npd-$(NODE_NAME).log 2>&1
|
||||
volumeMounts:
|
||||
- name: kubeconfig-volume
|
||||
mountPath: /kubeconfig
|
||||
readOnly: true
|
||||
- name: kernelmonitorconfig-volume
|
||||
mountPath: /config
|
||||
readOnly: true
|
||||
- name: no-serviceaccount-access-to-real-master
|
||||
mountPath: /var/run/secrets/kubernetes.io/serviceaccount
|
||||
readOnly: true
|
||||
- name: logs-volume
|
||||
mountPath: /var/log
|
||||
resources:
|
||||
requests:
|
||||
cpu: 20m
|
||||
memory: 20Mi
|
||||
securityContext:
|
||||
privileged: true
|
||||
@ -308,19 +308,19 @@ current-context: kubemark-context")
|
||||
"${KUBECTL}" create -f "${RESOURCE_DIRECTORY}/addons" --namespace="kubemark"
|
||||
|
||||
# Create the replication controller for hollow-nodes.
|
||||
sed "s/{{numreplicas}}/${NUM_NODES:-10}/g" "${RESOURCE_DIRECTORY}/hollow-node_template.json" > "${RESOURCE_DIRECTORY}/hollow-node.json"
|
||||
sed "s/{{numreplicas}}/${NUM_NODES:-10}/g" "${RESOURCE_DIRECTORY}/hollow-node_template.yaml" > "${RESOURCE_DIRECTORY}/hollow-node.yaml"
|
||||
proxy_cpu=20
|
||||
if [ "${NUM_NODES:-10}" -gt 1000 ]; then
|
||||
proxy_cpu=50
|
||||
fi
|
||||
proxy_mem_per_node=100
|
||||
proxy_mem=$((100 * 1024 + ${proxy_mem_per_node}*${NUM_NODES:-10}))
|
||||
sed -i'' -e "s/{{HOLLOW_PROXY_CPU}}/${proxy_cpu}/g" "${RESOURCE_DIRECTORY}/hollow-node.json"
|
||||
sed -i'' -e "s/{{HOLLOW_PROXY_MEM}}/${proxy_mem}/g" "${RESOURCE_DIRECTORY}/hollow-node.json"
|
||||
sed -i'' -e "s/{{registry}}/${CONTAINER_REGISTRY}/g" "${RESOURCE_DIRECTORY}/hollow-node.json"
|
||||
sed -i'' -e "s/{{project}}/${PROJECT}/g" "${RESOURCE_DIRECTORY}/hollow-node.json"
|
||||
sed -i'' -e "s/{{master_ip}}/${MASTER_IP}/g" "${RESOURCE_DIRECTORY}/hollow-node.json"
|
||||
"${KUBECTL}" create -f "${RESOURCE_DIRECTORY}/hollow-node.json" --namespace="kubemark"
|
||||
sed -i'' -e "s/{{HOLLOW_PROXY_CPU}}/${proxy_cpu}/g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
|
||||
sed -i'' -e "s/{{HOLLOW_PROXY_MEM}}/${proxy_mem}/g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
|
||||
sed -i'' -e "s/{{registry}}/${CONTAINER_REGISTRY}/g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
|
||||
sed -i'' -e "s/{{project}}/${PROJECT}/g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
|
||||
sed -i'' -e "s/{{master_ip}}/${MASTER_IP}/g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
|
||||
"${KUBECTL}" create -f "${RESOURCE_DIRECTORY}/hollow-node.yaml" --namespace="kubemark"
|
||||
|
||||
echo "Created secrets, configMaps, replication-controllers required for hollow-nodes."
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user