mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-24 12:15:52 +00:00
proxy/ipvs: Check that a dummy virtual server can be added
This tests both ipvs and the configured scheduler
This commit is contained in:
parent
4f02671b23
commit
cd15ca0548
@ -240,10 +240,10 @@ func newProxyServer(
|
|||||||
} else if proxyMode == proxyconfigapi.ProxyModeIPVS {
|
} else if proxyMode == proxyconfigapi.ProxyModeIPVS {
|
||||||
kernelHandler := ipvs.NewLinuxKernelHandler()
|
kernelHandler := ipvs.NewLinuxKernelHandler()
|
||||||
ipsetInterface = utilipset.New(execer)
|
ipsetInterface = utilipset.New(execer)
|
||||||
if err := ipvs.CanUseIPVSProxier(kernelHandler, ipsetInterface, config.IPVS.Scheduler); err != nil {
|
ipvsInterface = utilipvs.New()
|
||||||
|
if err := ipvs.CanUseIPVSProxier(ipvsInterface, ipsetInterface, config.IPVS.Scheduler); err != nil {
|
||||||
return nil, fmt.Errorf("can't use the IPVS proxier: %v", err)
|
return nil, fmt.Errorf("can't use the IPVS proxier: %v", err)
|
||||||
}
|
}
|
||||||
ipvsInterface = utilipvs.New()
|
|
||||||
|
|
||||||
klog.InfoS("Using ipvs Proxier")
|
klog.InfoS("Using ipvs Proxier")
|
||||||
if dualStack {
|
if dualStack {
|
||||||
|
@ -720,8 +720,60 @@ func (handle *LinuxKernelHandler) GetKernelVersion() (string, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// CanUseIPVSProxier checks if we can use the ipvs Proxier.
|
// CanUseIPVSProxier checks if we can use the ipvs Proxier.
|
||||||
// Only the ipset version is checked. Necessary kernel functions are assumed to be present.
|
// If the ipvs already has virtual servers (VS) we assume this is a re-start and
|
||||||
func CanUseIPVSProxier(handle KernelHandler, ipsetver IPSetVersioner, scheduler string) error {
|
// skip the tests. If the ipvs is empty we try to add a VS with the passed scheduler.
|
||||||
|
// If that works we assume that ipvs and the passed scheduler are supported.
|
||||||
|
// The ipset version is also checked.
|
||||||
|
func CanUseIPVSProxier(ipvs utilipvs.Interface, ipsetver IPSetVersioner, scheduler string) error {
|
||||||
|
// Check is any virtual servers (vs) are configured. If any, we assume
|
||||||
|
// that this is a kube-proxy re-start and skip the checks.
|
||||||
|
vservers, err := ipvs.GetVirtualServers()
|
||||||
|
if err != nil {
|
||||||
|
klog.ErrorS(err, "Can't read the ipvs")
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
klog.V(5).InfoS("Virtual Servers", "count", len(vservers))
|
||||||
|
if len(vservers) > 0 {
|
||||||
|
klog.InfoS("Assuming kube-proxy re-start", "virtual-servers", len(vservers))
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// BUG: If ipvs is not compiled into the kernel ipvs.GetVirtualServers
|
||||||
|
// does not return an error. Instead also ipvs.AddVirtualServer returns OK
|
||||||
|
// (no error)!
|
||||||
|
|
||||||
|
// This is the first time kube-proxy is loaded on the node. Try to
|
||||||
|
// insert a dummy VS with the passed scheduler. The VIP address can be
|
||||||
|
// anything since no traffic will be routed to ipvs yet.
|
||||||
|
vs := utilipvs.VirtualServer{
|
||||||
|
Address: net.ParseIP("10.0.0.1"),
|
||||||
|
Protocol: "TCP",
|
||||||
|
Port: 20000,
|
||||||
|
Scheduler: scheduler,
|
||||||
|
}
|
||||||
|
if err := ipvs.AddVirtualServer(&vs); err != nil {
|
||||||
|
klog.ErrorS(err, "Could not create dummy VS", "scheduler", scheduler)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// To overcome the BUG described above we check that the VS is *really* added.
|
||||||
|
vservers, err = ipvs.GetVirtualServers()
|
||||||
|
if err != nil {
|
||||||
|
klog.ErrorS(err, "ipvs.GetVirtualServers")
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
klog.V(5).InfoS("Virtual Servers after adding dummy", "count", len(vservers))
|
||||||
|
if len(vservers) == 0 {
|
||||||
|
klog.InfoS("Dummy VS not created", "scheduler", scheduler)
|
||||||
|
return fmt.Errorf("Ipvs not supported")
|
||||||
|
}
|
||||||
|
|
||||||
|
klog.V(5).InfoS("Dummy VS created", "vs", vs)
|
||||||
|
if err := ipvs.DeleteVirtualServer(&vs); err != nil {
|
||||||
|
klog.ErrorS(err, "Could not delete dummy VS")
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
// Check ipset version
|
// Check ipset version
|
||||||
versionString, err := ipsetver.GetVersion()
|
versionString, err := ipsetver.GetVersion()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -75,21 +75,60 @@ func (f *fakeIPGetter) BindedIPs() (sets.String, error) {
|
|||||||
return f.bindedIPs, nil
|
return f.bindedIPs, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// fakeKernelHandler implements KernelHandler.
|
// fakeIpvs implements utilipvs.Interface
|
||||||
type fakeKernelHandler struct {
|
type fakeIpvs struct {
|
||||||
modules []string
|
ipvsErr string
|
||||||
kernelVersion string
|
vsCreated bool
|
||||||
|
}
|
||||||
|
func (f *fakeIpvs) Flush() error {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
func (f *fakeIpvs) AddVirtualServer(*utilipvs.VirtualServer) error {
|
||||||
|
if f.ipvsErr == "AddVirtualServer" {
|
||||||
|
return fmt.Errorf("oops")
|
||||||
|
}
|
||||||
|
f.vsCreated = true
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
func (f *fakeIpvs) UpdateVirtualServer(*utilipvs.VirtualServer) error {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
func (f *fakeIpvs) DeleteVirtualServer(*utilipvs.VirtualServer) error {
|
||||||
|
if f.ipvsErr == "DeleteVirtualServer" {
|
||||||
|
return fmt.Errorf("oops")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
func (f *fakeIpvs) GetVirtualServer(*utilipvs.VirtualServer) (*utilipvs.VirtualServer, error) {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
func (f *fakeIpvs) GetVirtualServers() ([]*utilipvs.VirtualServer, error) {
|
||||||
|
if f.ipvsErr == "GetVirtualServers" {
|
||||||
|
return nil, fmt.Errorf("oops")
|
||||||
|
}
|
||||||
|
if f.vsCreated {
|
||||||
|
vs := []*utilipvs.VirtualServer{&utilipvs.VirtualServer{}}
|
||||||
|
return vs, nil
|
||||||
|
}
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
func (f *fakeIpvs) AddRealServer(*utilipvs.VirtualServer, *utilipvs.RealServer) error {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
func (f *fakeIpvs) GetRealServers(*utilipvs.VirtualServer) ([]*utilipvs.RealServer, error) {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
func (f *fakeIpvs) DeleteRealServer(*utilipvs.VirtualServer, *utilipvs.RealServer) error {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
func (f *fakeIpvs) UpdateRealServer(*utilipvs.VirtualServer, *utilipvs.RealServer) error {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
func (f *fakeIpvs) ConfigureTimeouts(time.Duration, time.Duration, time.Duration) error {
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (fake *fakeKernelHandler) GetModules() ([]string, error) {
|
// fakeIPSetVersioner implements IPSetVersioner.
|
||||||
return fake.modules, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (fake *fakeKernelHandler) GetKernelVersion() (string, error) {
|
|
||||||
return fake.kernelVersion, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// fakeKernelHandler implements KernelHandler.
|
|
||||||
type fakeIPSetVersioner struct {
|
type fakeIPSetVersioner struct {
|
||||||
version string
|
version string
|
||||||
err error
|
err error
|
||||||
@ -273,88 +312,57 @@ func TestCleanupLeftovers(t *testing.T) {
|
|||||||
|
|
||||||
func TestCanUseIPVSProxier(t *testing.T) {
|
func TestCanUseIPVSProxier(t *testing.T) {
|
||||||
testCases := []struct {
|
testCases := []struct {
|
||||||
mods []string
|
name string
|
||||||
scheduler string
|
scheduler string
|
||||||
kernelVersion string
|
|
||||||
kernelErr error
|
|
||||||
ipsetVersion string
|
ipsetVersion string
|
||||||
ipsetErr error
|
ipsetErr error
|
||||||
|
ipvsErr string
|
||||||
ok bool
|
ok bool
|
||||||
}{
|
}{
|
||||||
// case 0, kernel error
|
|
||||||
{
|
{
|
||||||
mods: []string{"foo", "bar", "baz"},
|
name: "happy days",
|
||||||
scheduler: "",
|
ipsetVersion: MinIPSetCheckVersion,
|
||||||
kernelVersion: "4.19",
|
ok: true,
|
||||||
kernelErr: fmt.Errorf("oops"),
|
|
||||||
ipsetVersion: "0.0",
|
|
||||||
ok: false,
|
|
||||||
},
|
},
|
||||||
// case 1, ipset error
|
|
||||||
{
|
{
|
||||||
mods: []string{"foo", "bar", "baz"},
|
name: "ipset error",
|
||||||
scheduler: "",
|
scheduler: "",
|
||||||
kernelVersion: "4.19",
|
|
||||||
ipsetVersion: MinIPSetCheckVersion,
|
ipsetVersion: MinIPSetCheckVersion,
|
||||||
ipsetErr: fmt.Errorf("oops"),
|
ipsetErr: fmt.Errorf("oops"),
|
||||||
ok: false,
|
ok: false,
|
||||||
},
|
},
|
||||||
// case 2, missing required kernel modules and ipset version too low
|
|
||||||
{
|
{
|
||||||
mods: []string{"foo", "bar", "baz"},
|
name: "ipset version too low",
|
||||||
scheduler: "rr",
|
scheduler: "rr",
|
||||||
kernelVersion: "4.19",
|
|
||||||
ipsetVersion: "1.1",
|
|
||||||
ok: false,
|
|
||||||
},
|
|
||||||
// case 4, ipset version too low
|
|
||||||
{
|
|
||||||
mods: []string{"ip_vs", "ip_vs_rr", "ip_vs_wrr", "ip_vs_sh", "nf_conntrack"},
|
|
||||||
scheduler: "rr",
|
|
||||||
kernelVersion: "4.19",
|
|
||||||
ipsetVersion: "4.3.0",
|
ipsetVersion: "4.3.0",
|
||||||
ok: false,
|
ok: false,
|
||||||
},
|
},
|
||||||
// case 5, ok for linux kernel 4.19
|
|
||||||
{
|
{
|
||||||
mods: []string{"ip_vs", "ip_vs_rr", "ip_vs_wrr", "ip_vs_sh", "nf_conntrack"},
|
name: "GetVirtualServers fail",
|
||||||
scheduler: "rr",
|
|
||||||
kernelVersion: "4.19",
|
|
||||||
ipsetVersion: MinIPSetCheckVersion,
|
ipsetVersion: MinIPSetCheckVersion,
|
||||||
ok: true,
|
ipvsErr: "GetVirtualServers",
|
||||||
|
ok: false,
|
||||||
},
|
},
|
||||||
// case 6, ok for linux kernel 4.18
|
|
||||||
{
|
{
|
||||||
mods: []string{"ip_vs", "ip_vs_rr", "ip_vs_wrr", "ip_vs_sh", "nf_conntrack_ipv4"},
|
name: "AddVirtualServer fail",
|
||||||
scheduler: "rr",
|
|
||||||
kernelVersion: "4.18",
|
|
||||||
ipsetVersion: MinIPSetCheckVersion,
|
ipsetVersion: MinIPSetCheckVersion,
|
||||||
ok: true,
|
ipvsErr: "AddVirtualServer",
|
||||||
|
ok: false,
|
||||||
},
|
},
|
||||||
// case 7. ok when module list has extra modules
|
|
||||||
{
|
{
|
||||||
mods: []string{"foo", "ip_vs", "ip_vs_rr", "ip_vs_wrr", "ip_vs_sh", "nf_conntrack", "bar"},
|
name: "DeleteVirtualServer fail",
|
||||||
scheduler: "rr",
|
|
||||||
kernelVersion: "4.19",
|
|
||||||
ipsetVersion: "6.19",
|
|
||||||
ok: true,
|
|
||||||
},
|
|
||||||
// case 9, ok for dh based IPVS scheduling
|
|
||||||
{
|
|
||||||
mods: []string{"ip_vs", "ip_vs_rr", "ip_vs_wrr", "ip_vs_sh", "nf_conntrack", "ip_vs_dh"},
|
|
||||||
scheduler: "dh",
|
|
||||||
kernelVersion: "4.19",
|
|
||||||
ipsetVersion: MinIPSetCheckVersion,
|
ipsetVersion: MinIPSetCheckVersion,
|
||||||
ok: true,
|
ipvsErr: "DeleteVirtualServer",
|
||||||
|
ok: false,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
for i := range testCases {
|
for _, tc := range testCases {
|
||||||
handle := &fakeKernelHandler{modules: testCases[i].mods, kernelVersion: testCases[i].kernelVersion}
|
ipvs := &fakeIpvs{tc.ipvsErr, false}
|
||||||
versioner := &fakeIPSetVersioner{version: testCases[i].ipsetVersion, err: testCases[i].ipsetErr}
|
versioner := &fakeIPSetVersioner{version: tc.ipsetVersion, err: tc.ipsetErr}
|
||||||
err := CanUseIPVSProxier(handle, versioner, testCases[i].scheduler)
|
err := CanUseIPVSProxier(ipvs, versioner, tc.scheduler)
|
||||||
if (err == nil) != testCases[i].ok {
|
if (err == nil) != tc.ok {
|
||||||
t.Errorf("Case [%d], expect %v, got err: %v", i, testCases[i].ok, err)
|
t.Errorf("Case [%s], expect %v, got err: %v", tc.name, tc.ok, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
Loading…
Reference in New Issue
Block a user