mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-24 12:15:52 +00:00
Merge pull request #42201 from shyamjvs/inotify-limit
Automatic merge from submit-queue (batch tested with PRs 42316, 41618, 42201, 42113, 42191) [Kubemark] Add init container in hollow node for setting inotify limit of node to 200 Fixes #41713 Along with adding the init container, I also changed the manifest to a yaml as otherwise the entire init container annotation would have to be in a single line (with escaped characters), as json doesn't allow multi-line strings. cc @kubernetes/sig-scalability-misc @wojtek-t @gmarek @Random-Liu
This commit is contained in:
Kubernetes Submit Queue
GitHub
commit
32d59cbb2f
@ -1,198 +0,0 @@
|
|||||||
{
|
|
||||||
"kind": "ReplicationController",
|
|
||||||
"apiVersion": "v1",
|
|
||||||
"metadata": {
|
|
||||||
"name": "hollow-node",
|
|
||||||
"labels": {
|
|
||||||
"name": "hollow-node"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"spec": {
|
|
||||||
"replicas": {{numreplicas}},
|
|
||||||
"selector": {
|
|
||||||
"name": "hollow-node"
|
|
||||||
},
|
|
||||||
"template": {
|
|
||||||
"metadata": {
|
|
||||||
"labels": {
|
|
||||||
"name": "hollow-node"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"spec": {
|
|
||||||
"volumes": [
|
|
||||||
{
|
|
||||||
"name": "kubeconfig-volume",
|
|
||||||
"secret": {
|
|
||||||
"secretName": "kubeconfig"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "kernelmonitorconfig-volume",
|
|
||||||
"configMap": {
|
|
||||||
"name": "node-configmap"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "logs-volume",
|
|
||||||
"hostPath": {
|
|
||||||
"path": "/var/log"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "no-serviceaccount-access-to-real-master",
|
|
||||||
"emptyDir": {}
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"containers": [
|
|
||||||
{
|
|
||||||
"name": "hollow-kubelet",
|
|
||||||
"image": "{{registry}}/{{project}}/kubemark:latest",
|
|
||||||
"ports": [
|
|
||||||
{"containerPort": 4194},
|
|
||||||
{"containerPort": 10250},
|
|
||||||
{"containerPort": 10255}
|
|
||||||
],
|
|
||||||
"env": [
|
|
||||||
{
|
|
||||||
"name": "CONTENT_TYPE",
|
|
||||||
"valueFrom": {
|
|
||||||
"configMapKeyRef": {
|
|
||||||
"name": "node-configmap",
|
|
||||||
"key": "content.type"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "NODE_NAME",
|
|
||||||
"valueFrom": {
|
|
||||||
"fieldRef": {
|
|
||||||
"fieldPath": "metadata.name"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"command": [
|
|
||||||
"/bin/sh",
|
|
||||||
"-c",
|
|
||||||
"./kubemark.sh --morph=kubelet --name=$(NODE_NAME) --kubeconfig=/kubeconfig/kubelet.kubeconfig $(CONTENT_TYPE) --v=2 1>>/var/log/kubelet-$(NODE_NAME).log 2>&1"
|
|
||||||
],
|
|
||||||
"volumeMounts": [
|
|
||||||
{
|
|
||||||
"name": "kubeconfig-volume",
|
|
||||||
"mountPath": "/kubeconfig",
|
|
||||||
"readOnly": true
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "logs-volume",
|
|
||||||
"mountPath": "/var/log"
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"resources": {
|
|
||||||
"requests": {
|
|
||||||
"cpu": "50m",
|
|
||||||
"memory": "100M"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"securityContext": {
|
|
||||||
"privileged": true
|
|
||||||
}
|
|
||||||
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "hollow-proxy",
|
|
||||||
"image": "{{registry}}/{{project}}/kubemark:latest",
|
|
||||||
"env": [
|
|
||||||
{
|
|
||||||
"name": "CONTENT_TYPE",
|
|
||||||
"valueFrom": {
|
|
||||||
"configMapKeyRef": {
|
|
||||||
"name": "node-configmap",
|
|
||||||
"key": "content.type"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "NODE_NAME",
|
|
||||||
"valueFrom": {
|
|
||||||
"fieldRef": {
|
|
||||||
"fieldPath": "metadata.name"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"command": [
|
|
||||||
"/bin/sh",
|
|
||||||
"-c",
|
|
||||||
"./kubemark.sh --morph=proxy --name=$(NODE_NAME) --kubeconfig=/kubeconfig/kubeproxy.kubeconfig $(CONTENT_TYPE) --v=2 1>>/var/log/kubeproxy-$(NODE_NAME).log 2>&1"
|
|
||||||
],
|
|
||||||
"volumeMounts": [
|
|
||||||
{
|
|
||||||
"name": "kubeconfig-volume",
|
|
||||||
"mountPath": "/kubeconfig",
|
|
||||||
"readOnly": true
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "logs-volume",
|
|
||||||
"mountPath": "/var/log"
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"resources": {
|
|
||||||
"requests": {
|
|
||||||
"cpu": "{{HOLLOW_PROXY_CPU}}m",
|
|
||||||
"memory": "{{HOLLOW_PROXY_MEM}}Ki"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "hollow-node-problem-detector",
|
|
||||||
"image": "gcr.io/google_containers/node-problem-detector:v0.3.0-alpha.0",
|
|
||||||
"env": [
|
|
||||||
{
|
|
||||||
"name": "NODE_NAME",
|
|
||||||
"valueFrom": {
|
|
||||||
"fieldRef": {
|
|
||||||
"fieldPath": "metadata.name"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"command": [
|
|
||||||
"/bin/sh",
|
|
||||||
"-c",
|
|
||||||
"/node-problem-detector --kernel-monitor=/config/kernel.monitor --apiserver-override=\"https://{{master_ip}}:443?inClusterConfig=false&auth=/kubeconfig/npd.kubeconfig\" --alsologtostderr 1>>/var/log/npd-$(NODE_NAME).log 2>&1"
|
|
||||||
],
|
|
||||||
"volumeMounts": [
|
|
||||||
{
|
|
||||||
"name": "kubeconfig-volume",
|
|
||||||
"mountPath": "/kubeconfig",
|
|
||||||
"readOnly": true
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "kernelmonitorconfig-volume",
|
|
||||||
"mountPath": "/config",
|
|
||||||
"readOnly": true
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "no-serviceaccount-access-to-real-master",
|
|
||||||
"mountPath": "/var/run/secrets/kubernetes.io/serviceaccount",
|
|
||||||
"readOnly": true
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "logs-volume",
|
|
||||||
"mountPath": "/var/log"
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"resources": {
|
|
||||||
"requests": {
|
|
||||||
"cpu": "20m",
|
|
||||||
"memory": "20Mi"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"securityContext": {
|
|
||||||
"privileged": true
|
|
||||||
}
|
|
||||||
}]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
126
test/kubemark/resources/hollow-node_template.yaml
Normal file
126
test/kubemark/resources/hollow-node_template.yaml
Normal file
@ -0,0 +1,126 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: ReplicationController
|
||||||
|
metadata:
|
||||||
|
name: hollow-node
|
||||||
|
labels:
|
||||||
|
name: hollow-node
|
||||||
|
spec:
|
||||||
|
replicas: {{numreplicas}}
|
||||||
|
selector:
|
||||||
|
name: hollow-node
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
name: hollow-node
|
||||||
|
annotations:
|
||||||
|
pod.beta.kubernetes.io/init-containers: '[
|
||||||
|
{
|
||||||
|
"name": "init-inotify-limit",
|
||||||
|
"image": "busybox",
|
||||||
|
"command": ["sysctl", "-w", "fs.inotify.max_user_instances=200"],
|
||||||
|
"securityContext": {
|
||||||
|
"privileged": true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]'
|
||||||
|
spec:
|
||||||
|
volumes:
|
||||||
|
- name: kubeconfig-volume
|
||||||
|
secret:
|
||||||
|
secretName: kubeconfig
|
||||||
|
- name: kernelmonitorconfig-volume
|
||||||
|
configMap:
|
||||||
|
name: node-configmap
|
||||||
|
- name: logs-volume
|
||||||
|
hostPath:
|
||||||
|
path: /var/log
|
||||||
|
- name: no-serviceaccount-access-to-real-master
|
||||||
|
emptyDir: {}
|
||||||
|
containers:
|
||||||
|
- name: hollow-kubelet
|
||||||
|
image: {{registry}}/{{project}}/kubemark:latest
|
||||||
|
ports:
|
||||||
|
- containerPort: 4194
|
||||||
|
- containerPort: 10250
|
||||||
|
- containerPort: 10255
|
||||||
|
env:
|
||||||
|
- name: CONTENT_TYPE
|
||||||
|
valueFrom:
|
||||||
|
configMapKeyRef:
|
||||||
|
name: node-configmap
|
||||||
|
key: content.type
|
||||||
|
- name: NODE_NAME
|
||||||
|
valueFrom:
|
||||||
|
fieldRef:
|
||||||
|
fieldPath: metadata.name
|
||||||
|
command:
|
||||||
|
- /bin/sh
|
||||||
|
- -c
|
||||||
|
- ./kubemark.sh --morph=kubelet --name=$(NODE_NAME) --kubeconfig=/kubeconfig/kubelet.kubeconfig $(CONTENT_TYPE) --v=2 1>>/var/log/kubelet-$(NODE_NAME).log 2>&1
|
||||||
|
volumeMounts:
|
||||||
|
- name: kubeconfig-volume
|
||||||
|
mountPath: /kubeconfig
|
||||||
|
readOnly: true
|
||||||
|
- name: logs-volume
|
||||||
|
mountPath: /var/log
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
cpu: 50m
|
||||||
|
memory: 100M
|
||||||
|
securityContext:
|
||||||
|
privileged: true
|
||||||
|
- name: hollow-proxy
|
||||||
|
image: {{registry}}/{{project}}/kubemark:latest
|
||||||
|
env:
|
||||||
|
- name: CONTENT_TYPE
|
||||||
|
valueFrom:
|
||||||
|
configMapKeyRef:
|
||||||
|
name: node-configmap
|
||||||
|
key: content.type
|
||||||
|
- name: NODE_NAME
|
||||||
|
valueFrom:
|
||||||
|
fieldRef:
|
||||||
|
fieldPath: metadata.name
|
||||||
|
command:
|
||||||
|
- /bin/sh
|
||||||
|
- -c
|
||||||
|
- ./kubemark.sh --morph=proxy --name=$(NODE_NAME) --kubeconfig=/kubeconfig/kubeproxy.kubeconfig $(CONTENT_TYPE) --v=2 1>>/var/log/kubeproxy-$(NODE_NAME).log 2>&1
|
||||||
|
volumeMounts:
|
||||||
|
- name: kubeconfig-volume
|
||||||
|
mountPath: /kubeconfig
|
||||||
|
readOnly: true
|
||||||
|
- name: logs-volume
|
||||||
|
mountPath: /var/log
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
cpu: {{HOLLOW_PROXY_CPU}}m
|
||||||
|
memory: {{HOLLOW_PROXY_MEM}}Ki
|
||||||
|
- name: hollow-node-problem-detector
|
||||||
|
image: gcr.io/google_containers/node-problem-detector:v0.3.0-alpha.0
|
||||||
|
env:
|
||||||
|
- name: NODE_NAME
|
||||||
|
valueFrom:
|
||||||
|
fieldRef:
|
||||||
|
fieldPath: metadata.name
|
||||||
|
command:
|
||||||
|
- /bin/sh
|
||||||
|
- -c
|
||||||
|
- /node-problem-detector --kernel-monitor=/config/kernel.monitor --apiserver-override="https://{{master_ip}}:443?inClusterConfig=false&auth=/kubeconfig/npd.kubeconfig" --alsologtostderr 1>>/var/log/npd-$(NODE_NAME).log 2>&1
|
||||||
|
volumeMounts:
|
||||||
|
- name: kubeconfig-volume
|
||||||
|
mountPath: /kubeconfig
|
||||||
|
readOnly: true
|
||||||
|
- name: kernelmonitorconfig-volume
|
||||||
|
mountPath: /config
|
||||||
|
readOnly: true
|
||||||
|
- name: no-serviceaccount-access-to-real-master
|
||||||
|
mountPath: /var/run/secrets/kubernetes.io/serviceaccount
|
||||||
|
readOnly: true
|
||||||
|
- name: logs-volume
|
||||||
|
mountPath: /var/log
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
cpu: 20m
|
||||||
|
memory: 20Mi
|
||||||
|
securityContext:
|
||||||
|
privileged: true
|
||||||
@ -308,19 +308,19 @@ current-context: kubemark-context")
|
|||||||
"${KUBECTL}" create -f "${RESOURCE_DIRECTORY}/addons" --namespace="kubemark"
|
"${KUBECTL}" create -f "${RESOURCE_DIRECTORY}/addons" --namespace="kubemark"
|
||||||
|
|
||||||
# Create the replication controller for hollow-nodes.
|
# Create the replication controller for hollow-nodes.
|
||||||
sed "s/{{numreplicas}}/${NUM_NODES:-10}/g" "${RESOURCE_DIRECTORY}/hollow-node_template.json" > "${RESOURCE_DIRECTORY}/hollow-node.json"
|
sed "s/{{numreplicas}}/${NUM_NODES:-10}/g" "${RESOURCE_DIRECTORY}/hollow-node_template.yaml" > "${RESOURCE_DIRECTORY}/hollow-node.yaml"
|
||||||
proxy_cpu=20
|
proxy_cpu=20
|
||||||
if [ "${NUM_NODES:-10}" -gt 1000 ]; then
|
if [ "${NUM_NODES:-10}" -gt 1000 ]; then
|
||||||
proxy_cpu=50
|
proxy_cpu=50
|
||||||
fi
|
fi
|
||||||
proxy_mem_per_node=100
|
proxy_mem_per_node=100
|
||||||
proxy_mem=$((100 * 1024 + ${proxy_mem_per_node}*${NUM_NODES:-10}))
|
proxy_mem=$((100 * 1024 + ${proxy_mem_per_node}*${NUM_NODES:-10}))
|
||||||
sed -i'' -e "s/{{HOLLOW_PROXY_CPU}}/${proxy_cpu}/g" "${RESOURCE_DIRECTORY}/hollow-node.json"
|
sed -i'' -e "s/{{HOLLOW_PROXY_CPU}}/${proxy_cpu}/g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
|
||||||
sed -i'' -e "s/{{HOLLOW_PROXY_MEM}}/${proxy_mem}/g" "${RESOURCE_DIRECTORY}/hollow-node.json"
|
sed -i'' -e "s/{{HOLLOW_PROXY_MEM}}/${proxy_mem}/g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
|
||||||
sed -i'' -e "s/{{registry}}/${CONTAINER_REGISTRY}/g" "${RESOURCE_DIRECTORY}/hollow-node.json"
|
sed -i'' -e "s/{{registry}}/${CONTAINER_REGISTRY}/g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
|
||||||
sed -i'' -e "s/{{project}}/${PROJECT}/g" "${RESOURCE_DIRECTORY}/hollow-node.json"
|
sed -i'' -e "s/{{project}}/${PROJECT}/g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
|
||||||
sed -i'' -e "s/{{master_ip}}/${MASTER_IP}/g" "${RESOURCE_DIRECTORY}/hollow-node.json"
|
sed -i'' -e "s/{{master_ip}}/${MASTER_IP}/g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
|
||||||
"${KUBECTL}" create -f "${RESOURCE_DIRECTORY}/hollow-node.json" --namespace="kubemark"
|
"${KUBECTL}" create -f "${RESOURCE_DIRECTORY}/hollow-node.yaml" --namespace="kubemark"
|
||||||
|
|
||||||
echo "Created secrets, configMaps, replication-controllers required for hollow-nodes."
|
echo "Created secrets, configMaps, replication-controllers required for hollow-nodes."
|
||||||
}
|
}
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user