mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-22 11:21:47 +00:00
Prevent kube-proxy from panicing when sysfs is mounted as read-only.
Send a node event when this happens and hint to the administrator about the remediation.
This commit is contained in:
parent
42805f53b0
commit
4246853211
@ -17,11 +17,13 @@ limitations under the License.
|
|||||||
package app
|
package app
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"errors"
|
||||||
"io/ioutil"
|
"io/ioutil"
|
||||||
"strconv"
|
"strconv"
|
||||||
|
|
||||||
"github.com/golang/glog"
|
"github.com/golang/glog"
|
||||||
|
|
||||||
|
"k8s.io/kubernetes/pkg/util/mount"
|
||||||
"k8s.io/kubernetes/pkg/util/sysctl"
|
"k8s.io/kubernetes/pkg/util/sysctl"
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -32,11 +34,25 @@ type Conntracker interface {
|
|||||||
|
|
||||||
type realConntracker struct{}
|
type realConntracker struct{}
|
||||||
|
|
||||||
|
var readOnlySysFSError = errors.New("ReadOnlySysFS")
|
||||||
|
|
||||||
func (realConntracker) SetMax(max int) error {
|
func (realConntracker) SetMax(max int) error {
|
||||||
glog.Infof("Setting nf_conntrack_max to %d", max)
|
glog.Infof("Setting nf_conntrack_max to %d", max)
|
||||||
if err := sysctl.SetSysctl("net/netfilter/nf_conntrack_max", max); err != nil {
|
if err := sysctl.SetSysctl("net/netfilter/nf_conntrack_max", max); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
// sysfs is expected to be mounted as 'rw'. However, it may be unexpectedly mounted as
|
||||||
|
// 'ro' by docker because of a known docker issue (https://github.com/docker/docker/issues/24000).
|
||||||
|
// Setting conntrack will fail when sysfs is readonly. When that happens, we don't set conntrack
|
||||||
|
// hashsize and return a special error readOnlySysFSError here. The caller should deal with
|
||||||
|
// readOnlySysFSError differently.
|
||||||
|
writable, err := isSysFSWritable()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if !writable {
|
||||||
|
return readOnlySysFSError
|
||||||
|
}
|
||||||
// TODO: generify this and sysctl to a new sysfs.WriteInt()
|
// TODO: generify this and sysctl to a new sysfs.WriteInt()
|
||||||
glog.Infof("Setting conntrack hashsize to %d", max/4)
|
glog.Infof("Setting conntrack hashsize to %d", max/4)
|
||||||
return ioutil.WriteFile("/sys/module/nf_conntrack/parameters/hashsize", []byte(strconv.Itoa(max/4)), 0640)
|
return ioutil.WriteFile("/sys/module/nf_conntrack/parameters/hashsize", []byte(strconv.Itoa(max/4)), 0640)
|
||||||
@ -46,3 +62,27 @@ func (realConntracker) SetTCPEstablishedTimeout(seconds int) error {
|
|||||||
glog.Infof("Setting nf_conntrack_tcp_timeout_established to %d", seconds)
|
glog.Infof("Setting nf_conntrack_tcp_timeout_established to %d", seconds)
|
||||||
return sysctl.SetSysctl("net/netfilter/nf_conntrack_tcp_timeout_established", seconds)
|
return sysctl.SetSysctl("net/netfilter/nf_conntrack_tcp_timeout_established", seconds)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// isSysFSWritable checks /proc/mounts to see whether sysfs is 'rw' or not.
|
||||||
|
func isSysFSWritable() (bool, error) {
|
||||||
|
const permWritable = "rw"
|
||||||
|
const sysfsDevice = "sysfs"
|
||||||
|
m := mount.New()
|
||||||
|
mountPoints, err := m.List()
|
||||||
|
if err != nil {
|
||||||
|
glog.Errorf("failed to list mount points: %v", err)
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
for _, mountPoint := range mountPoints {
|
||||||
|
if mountPoint.Device != sysfsDevice {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
// Check whether sysfs is 'rw'
|
||||||
|
if len(mountPoint.Opts) > 0 && mountPoint.Opts[0] == permWritable {
|
||||||
|
return true, nil
|
||||||
|
}
|
||||||
|
glog.Errorf("sysfs is not writable: %+v", mountPoint)
|
||||||
|
break
|
||||||
|
}
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
@ -299,8 +299,19 @@ func (s *ProxyServer) Run() error {
|
|||||||
// Tune conntrack, if requested
|
// Tune conntrack, if requested
|
||||||
if s.Conntracker != nil {
|
if s.Conntracker != nil {
|
||||||
if s.Config.ConntrackMax > 0 {
|
if s.Config.ConntrackMax > 0 {
|
||||||
if err := s.Conntracker.SetMax(int(s.Config.ConntrackMax)); err != nil {
|
err := s.Conntracker.SetMax(int(s.Config.ConntrackMax))
|
||||||
return err
|
if err != nil {
|
||||||
|
if err != readOnlySysFSError {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
// readOnlySysFSError is caused by a known docker issue (https://github.com/docker/docker/issues/24000),
|
||||||
|
// the only remediation we know is to restart the docker daemon.
|
||||||
|
// Here we'll send an node event with specific reason and message, the
|
||||||
|
// administrator should decide whether and how to handle this issue,
|
||||||
|
// whether to drain the node and restart docker.
|
||||||
|
// TODO(random-liu): Remove this when the docker bug is fixed.
|
||||||
|
const message = "DOCKER RESTART NEEDED (docker issue #24000): /sys is read-only: can't raise conntrack limits, problems may arise later."
|
||||||
|
s.Recorder.Eventf(s.Config.NodeRef, api.EventTypeWarning, err.Error(), message)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if s.Config.ConntrackTCPEstablishedTimeout.Duration > 0 {
|
if s.Config.ConntrackTCPEstablishedTimeout.Duration > 0 {
|
||||||
|
Loading…
Reference in New Issue
Block a user