mirror of
https://github.com/k3s-io/kubernetes.git
synced 2026-01-13 11:25:19 +00:00
Remove conntrack entry on udp rule add.
Moved conntrack util outside of proxy pkg Added warning message if conntrack binary is not found Addressed review comments. ran gofmt
This commit is contained in:
Pavithra Ramesh
@@ -15,6 +15,7 @@ filegroup(
|
||||
"//pkg/util/bandwidth:all-srcs",
|
||||
"//pkg/util/config:all-srcs",
|
||||
"//pkg/util/configz:all-srcs",
|
||||
"//pkg/util/conntrack:all-srcs",
|
||||
"//pkg/util/dbus:all-srcs",
|
||||
"//pkg/util/ebtables:all-srcs",
|
||||
"//pkg/util/env:all-srcs",
|
||||
|
||||
41
pkg/util/conntrack/BUILD
Normal file
41
pkg/util/conntrack/BUILD
Normal file
@@ -0,0 +1,41 @@
|
||||
load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
|
||||
|
||||
go_library(
|
||||
name = "go_default_library",
|
||||
srcs = [
|
||||
"conntrack.go",
|
||||
],
|
||||
importpath = "k8s.io/kubernetes/pkg/util/conntrack",
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
"//vendor/k8s.io/api/core/v1:go_default_library",
|
||||
"//vendor/k8s.io/utils/exec:go_default_library",
|
||||
],
|
||||
)
|
||||
|
||||
go_test(
|
||||
name = "go_default_test",
|
||||
srcs = [
|
||||
"conntrack_test.go",
|
||||
],
|
||||
embed = [":go_default_library"],
|
||||
deps = [
|
||||
"//vendor/k8s.io/api/core/v1:go_default_library",
|
||||
"//vendor/k8s.io/utils/exec:go_default_library",
|
||||
"//vendor/k8s.io/utils/exec/testing:go_default_library",
|
||||
],
|
||||
)
|
||||
|
||||
filegroup(
|
||||
name = "package-srcs",
|
||||
srcs = glob(["**"]),
|
||||
tags = ["automanaged"],
|
||||
visibility = ["//visibility:private"],
|
||||
)
|
||||
|
||||
filegroup(
|
||||
name = "all-srcs",
|
||||
srcs = [":package-srcs"],
|
||||
tags = ["automanaged"],
|
||||
visibility = ["//visibility:public"],
|
||||
)
|
||||
120
pkg/util/conntrack/conntrack.go
Normal file
120
pkg/util/conntrack/conntrack.go
Normal file
@@ -0,0 +1,120 @@
|
||||
/*
|
||||
Copyright 2016 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package conntrack
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"k8s.io/api/core/v1"
|
||||
"k8s.io/utils/exec"
|
||||
)
|
||||
|
||||
// Utilities for dealing with conntrack
|
||||
|
||||
// NoConnectionToDelete is the error string returned by conntrack when no matching connections are found
|
||||
const NoConnectionToDelete = "0 flow entries have been deleted"
|
||||
|
||||
// IsIPv6 returns true if the given ip address is a valid ipv6 address
|
||||
func IsIPv6(netIP net.IP) bool {
|
||||
return netIP != nil && netIP.To4() == nil
|
||||
}
|
||||
|
||||
// IsIPv6String returns true if the given string is a valid ipv6 address
|
||||
func IsIPv6String(ip string) bool {
|
||||
netIP := net.ParseIP(ip)
|
||||
return IsIPv6(netIP)
|
||||
}
|
||||
|
||||
func protoStr(proto v1.Protocol) string {
|
||||
return strings.ToLower(string(proto))
|
||||
}
|
||||
|
||||
func parametersWithFamily(isIPv6 bool, parameters ...string) []string {
|
||||
if isIPv6 {
|
||||
parameters = append(parameters, "-f", "ipv6")
|
||||
}
|
||||
return parameters
|
||||
}
|
||||
|
||||
// ClearEntriesForIP uses the conntrack tool to delete the conntrack entries
|
||||
// for the UDP connections specified by the given service IP
|
||||
func ClearEntriesForIP(execer exec.Interface, ip string, protocol v1.Protocol) error {
|
||||
parameters := parametersWithFamily(IsIPv6String(ip), "-D", "--orig-dst", ip, "-p", protoStr(protocol))
|
||||
err := Exec(execer, parameters...)
|
||||
if err != nil && !strings.Contains(err.Error(), NoConnectionToDelete) {
|
||||
// TODO: Better handling for deletion failure. When failure occur, stale udp connection may not get flushed.
|
||||
// These stale udp connection will keep black hole traffic. Making this a best effort operation for now, since it
|
||||
// is expensive to baby-sit all udp connections to kubernetes services.
|
||||
return fmt.Errorf("error deleting connection tracking state for UDP service IP: %s, error: %v", ip, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Exec executes the conntrack tool using the given parameters
|
||||
func Exec(execer exec.Interface, parameters ...string) error {
|
||||
conntrackPath, err := execer.LookPath("conntrack")
|
||||
if err != nil {
|
||||
return fmt.Errorf("error looking for path of conntrack: %v", err)
|
||||
}
|
||||
output, err := execer.Command(conntrackPath, parameters...).CombinedOutput()
|
||||
if err != nil {
|
||||
return fmt.Errorf("conntrack command returned: %q, error message: %s", string(output), err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Exists returns true if conntrack binary is installed.
|
||||
func Exists(execer exec.Interface) bool {
|
||||
_, err := execer.LookPath("conntrack")
|
||||
return err == nil
|
||||
}
|
||||
|
||||
// ClearEntriesForPort uses the conntrack tool to delete the conntrack entries
|
||||
// for connections specified by the port.
|
||||
// When a packet arrives, it will not go through NAT table again, because it is not "the first" packet.
|
||||
// The solution is clearing the conntrack. Known issues:
|
||||
// https://github.com/docker/docker/issues/8795
|
||||
// https://github.com/kubernetes/kubernetes/issues/31983
|
||||
func ClearEntriesForPort(execer exec.Interface, port int, isIPv6 bool, protocol v1.Protocol) error {
|
||||
if port <= 0 {
|
||||
return fmt.Errorf("Wrong port number. The port number must be greater than zero")
|
||||
}
|
||||
parameters := parametersWithFamily(isIPv6, "-D", "-p", protoStr(protocol), "--dport", strconv.Itoa(port))
|
||||
err := Exec(execer, parameters...)
|
||||
if err != nil && !strings.Contains(err.Error(), NoConnectionToDelete) {
|
||||
return fmt.Errorf("error deleting conntrack entries for UDP port: %d, error: %v", port, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ClearEntriesForNAT uses the conntrack tool to delete the conntrack entries
|
||||
// for connections specified by the {origin, dest} IP pair.
|
||||
func ClearEntriesForNAT(execer exec.Interface, origin, dest string, protocol v1.Protocol) error {
|
||||
parameters := parametersWithFamily(IsIPv6String(origin), "-D", "--orig-dst", origin, "--dst-nat", dest,
|
||||
"-p", protoStr(protocol))
|
||||
err := Exec(execer, parameters...)
|
||||
if err != nil && !strings.Contains(err.Error(), NoConnectionToDelete) {
|
||||
// TODO: Better handling for deletion failure. When failure occur, stale udp connection may not get flushed.
|
||||
// These stale udp connection will keep black hole traffic. Making this a best effort operation for now, since it
|
||||
// is expensive to baby sit all udp connections to kubernetes services.
|
||||
return fmt.Errorf("error deleting conntrack entries for UDP peer {%s, %s}, error: %v", origin, dest, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
332
pkg/util/conntrack/conntrack_test.go
Normal file
332
pkg/util/conntrack/conntrack_test.go
Normal file
@@ -0,0 +1,332 @@
|
||||
/*
|
||||
Copyright 2015 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package conntrack
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"k8s.io/api/core/v1"
|
||||
"k8s.io/utils/exec"
|
||||
fakeexec "k8s.io/utils/exec/testing"
|
||||
)
|
||||
|
||||
func familyParamStr(isIPv6 bool) string {
|
||||
if isIPv6 {
|
||||
return " -f ipv6"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func TestExecConntrackTool(t *testing.T) {
|
||||
fcmd := fakeexec.FakeCmd{
|
||||
CombinedOutputScript: []fakeexec.FakeCombinedOutputAction{
|
||||
func() ([]byte, error) { return []byte("1 flow entries have been deleted"), nil },
|
||||
func() ([]byte, error) { return []byte("1 flow entries have been deleted"), nil },
|
||||
func() ([]byte, error) {
|
||||
return []byte(""), fmt.Errorf("conntrack v1.4.2 (conntrack-tools): 0 flow entries have been deleted")
|
||||
},
|
||||
},
|
||||
}
|
||||
fexec := fakeexec.FakeExec{
|
||||
CommandScript: []fakeexec.FakeCommandAction{
|
||||
func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) },
|
||||
func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) },
|
||||
func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) },
|
||||
},
|
||||
LookPathFunc: func(cmd string) (string, error) { return cmd, nil },
|
||||
}
|
||||
|
||||
testCases := [][]string{
|
||||
{"-L", "-p", "udp"},
|
||||
{"-D", "-p", "udp", "-d", "10.0.240.1"},
|
||||
{"-D", "-p", "udp", "--orig-dst", "10.240.0.2", "--dst-nat", "10.0.10.2"},
|
||||
}
|
||||
|
||||
expectErr := []bool{false, false, true}
|
||||
|
||||
for i := range testCases {
|
||||
err := Exec(&fexec, testCases[i]...)
|
||||
|
||||
if expectErr[i] {
|
||||
if err == nil {
|
||||
t.Errorf("expected err, got %v", err)
|
||||
}
|
||||
} else {
|
||||
if err != nil {
|
||||
t.Errorf("expected success, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
execCmd := strings.Join(fcmd.CombinedOutputLog[i], " ")
|
||||
expectCmd := fmt.Sprintf("%s %s", "conntrack", strings.Join(testCases[i], " "))
|
||||
|
||||
if execCmd != expectCmd {
|
||||
t.Errorf("expect execute command: %s, but got: %s", expectCmd, execCmd)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestClearUDPConntrackForIP(t *testing.T) {
|
||||
fcmd := fakeexec.FakeCmd{
|
||||
CombinedOutputScript: []fakeexec.FakeCombinedOutputAction{
|
||||
func() ([]byte, error) { return []byte("1 flow entries have been deleted"), nil },
|
||||
func() ([]byte, error) { return []byte("1 flow entries have been deleted"), nil },
|
||||
func() ([]byte, error) {
|
||||
return []byte(""), fmt.Errorf("conntrack v1.4.2 (conntrack-tools): 0 flow entries have been deleted")
|
||||
},
|
||||
func() ([]byte, error) { return []byte("1 flow entries have been deleted"), nil },
|
||||
},
|
||||
}
|
||||
fexec := fakeexec.FakeExec{
|
||||
CommandScript: []fakeexec.FakeCommandAction{
|
||||
func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) },
|
||||
func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) },
|
||||
func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) },
|
||||
func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) },
|
||||
},
|
||||
LookPathFunc: func(cmd string) (string, error) { return cmd, nil },
|
||||
}
|
||||
|
||||
testCases := []struct {
|
||||
name string
|
||||
ip string
|
||||
}{
|
||||
{"IPv4 success", "10.240.0.3"},
|
||||
{"IPv4 success", "10.240.0.5"},
|
||||
{"IPv4 simulated error", "10.240.0.4"},
|
||||
{"IPv6 success", "2001:db8::10"},
|
||||
}
|
||||
|
||||
svcCount := 0
|
||||
for _, tc := range testCases {
|
||||
if err := ClearEntriesForIP(&fexec, tc.ip, v1.ProtocolUDP); err != nil {
|
||||
t.Errorf("%s test case:, Unexpected error: %v", tc.name, err)
|
||||
}
|
||||
expectCommand := fmt.Sprintf("conntrack -D --orig-dst %s -p udp", tc.ip) + familyParamStr(IsIPv6String(tc.ip))
|
||||
execCommand := strings.Join(fcmd.CombinedOutputLog[svcCount], " ")
|
||||
if expectCommand != execCommand {
|
||||
t.Errorf("%s test case: Expect command: %s, but executed %s", tc.name, expectCommand, execCommand)
|
||||
}
|
||||
svcCount++
|
||||
}
|
||||
if svcCount != fexec.CommandCalls {
|
||||
t.Errorf("Expect command executed %d times, but got %d", svcCount, fexec.CommandCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestClearUDPConntrackForPort(t *testing.T) {
|
||||
fcmd := fakeexec.FakeCmd{
|
||||
CombinedOutputScript: []fakeexec.FakeCombinedOutputAction{
|
||||
func() ([]byte, error) { return []byte("1 flow entries have been deleted"), nil },
|
||||
func() ([]byte, error) {
|
||||
return []byte(""), fmt.Errorf("conntrack v1.4.2 (conntrack-tools): 0 flow entries have been deleted")
|
||||
},
|
||||
func() ([]byte, error) { return []byte("1 flow entries have been deleted"), nil },
|
||||
},
|
||||
}
|
||||
fexec := fakeexec.FakeExec{
|
||||
CommandScript: []fakeexec.FakeCommandAction{
|
||||
func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) },
|
||||
func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) },
|
||||
func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) },
|
||||
},
|
||||
LookPathFunc: func(cmd string) (string, error) { return cmd, nil },
|
||||
}
|
||||
|
||||
testCases := []struct {
|
||||
name string
|
||||
port int
|
||||
isIPv6 bool
|
||||
}{
|
||||
{"IPv4, no error", 8080, false},
|
||||
{"IPv4, simulated error", 9090, false},
|
||||
{"IPv6, no error", 6666, true},
|
||||
}
|
||||
svcCount := 0
|
||||
for _, tc := range testCases {
|
||||
err := ClearEntriesForPort(&fexec, tc.port, tc.isIPv6, v1.ProtocolUDP)
|
||||
if err != nil {
|
||||
t.Errorf("%s test case: Unexpected error: %v", tc.name, err)
|
||||
}
|
||||
expectCommand := fmt.Sprintf("conntrack -D -p udp --dport %d", tc.port) + familyParamStr(tc.isIPv6)
|
||||
execCommand := strings.Join(fcmd.CombinedOutputLog[svcCount], " ")
|
||||
if expectCommand != execCommand {
|
||||
t.Errorf("%s test case: Expect command: %s, but executed %s", tc.name, expectCommand, execCommand)
|
||||
}
|
||||
svcCount++
|
||||
}
|
||||
if svcCount != fexec.CommandCalls {
|
||||
t.Errorf("Expect command executed %d times, but got %d", svcCount, fexec.CommandCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteUDPConnections(t *testing.T) {
|
||||
fcmd := fakeexec.FakeCmd{
|
||||
CombinedOutputScript: []fakeexec.FakeCombinedOutputAction{
|
||||
func() ([]byte, error) { return []byte("1 flow entries have been deleted"), nil },
|
||||
func() ([]byte, error) {
|
||||
return []byte(""), fmt.Errorf("conntrack v1.4.2 (conntrack-tools): 0 flow entries have been deleted")
|
||||
},
|
||||
func() ([]byte, error) { return []byte("1 flow entries have been deleted"), nil },
|
||||
},
|
||||
}
|
||||
fexec := fakeexec.FakeExec{
|
||||
CommandScript: []fakeexec.FakeCommandAction{
|
||||
func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) },
|
||||
func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) },
|
||||
func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) },
|
||||
},
|
||||
LookPathFunc: func(cmd string) (string, error) { return cmd, nil },
|
||||
}
|
||||
|
||||
testCases := []struct {
|
||||
name string
|
||||
origin string
|
||||
dest string
|
||||
}{
|
||||
{
|
||||
name: "IPv4 success",
|
||||
origin: "1.2.3.4",
|
||||
dest: "10.20.30.40",
|
||||
},
|
||||
{
|
||||
name: "IPv4 simulated failure",
|
||||
origin: "2.3.4.5",
|
||||
dest: "20.30.40.50",
|
||||
},
|
||||
{
|
||||
name: "IPv6 success",
|
||||
origin: "fd00::600d:f00d",
|
||||
dest: "2001:db8::5",
|
||||
},
|
||||
}
|
||||
svcCount := 0
|
||||
for i, tc := range testCases {
|
||||
err := ClearEntriesForNAT(&fexec, tc.origin, tc.dest, v1.ProtocolUDP)
|
||||
if err != nil {
|
||||
t.Errorf("%s test case: unexpected error: %v", tc.name, err)
|
||||
}
|
||||
expectCommand := fmt.Sprintf("conntrack -D --orig-dst %s --dst-nat %s -p udp", tc.origin, tc.dest) + familyParamStr(IsIPv6String(tc.origin))
|
||||
execCommand := strings.Join(fcmd.CombinedOutputLog[i], " ")
|
||||
if expectCommand != execCommand {
|
||||
t.Errorf("%s test case: Expect command: %s, but executed %s", tc.name, expectCommand, execCommand)
|
||||
}
|
||||
svcCount++
|
||||
}
|
||||
if svcCount != fexec.CommandCalls {
|
||||
t.Errorf("Expect command executed %d times, but got %d", svcCount, fexec.CommandCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsIPv6String(t *testing.T) {
|
||||
testCases := []struct {
|
||||
ip string
|
||||
expectIPv6 bool
|
||||
}{
|
||||
{
|
||||
ip: "127.0.0.1",
|
||||
expectIPv6: false,
|
||||
},
|
||||
{
|
||||
ip: "192.168.0.0",
|
||||
expectIPv6: false,
|
||||
},
|
||||
{
|
||||
ip: "1.2.3.4",
|
||||
expectIPv6: false,
|
||||
},
|
||||
{
|
||||
ip: "bad ip",
|
||||
expectIPv6: false,
|
||||
},
|
||||
{
|
||||
ip: "::1",
|
||||
expectIPv6: true,
|
||||
},
|
||||
{
|
||||
ip: "fd00::600d:f00d",
|
||||
expectIPv6: true,
|
||||
},
|
||||
{
|
||||
ip: "2001:db8::5",
|
||||
expectIPv6: true,
|
||||
},
|
||||
}
|
||||
for i := range testCases {
|
||||
isIPv6 := IsIPv6String(testCases[i].ip)
|
||||
if isIPv6 != testCases[i].expectIPv6 {
|
||||
t.Errorf("[%d] Expect ipv6 %v, got %v", i+1, testCases[i].expectIPv6, isIPv6)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsIPv6(t *testing.T) {
|
||||
testCases := []struct {
|
||||
ip net.IP
|
||||
expectIPv6 bool
|
||||
}{
|
||||
{
|
||||
ip: net.IPv4zero,
|
||||
expectIPv6: false,
|
||||
},
|
||||
{
|
||||
ip: net.IPv4bcast,
|
||||
expectIPv6: false,
|
||||
},
|
||||
{
|
||||
ip: net.ParseIP("127.0.0.1"),
|
||||
expectIPv6: false,
|
||||
},
|
||||
{
|
||||
ip: net.ParseIP("10.20.40.40"),
|
||||
expectIPv6: false,
|
||||
},
|
||||
{
|
||||
ip: net.ParseIP("172.17.3.0"),
|
||||
expectIPv6: false,
|
||||
},
|
||||
{
|
||||
ip: nil,
|
||||
expectIPv6: false,
|
||||
},
|
||||
{
|
||||
ip: net.IPv6loopback,
|
||||
expectIPv6: true,
|
||||
},
|
||||
{
|
||||
ip: net.IPv6zero,
|
||||
expectIPv6: true,
|
||||
},
|
||||
{
|
||||
ip: net.ParseIP("fd00::600d:f00d"),
|
||||
expectIPv6: true,
|
||||
},
|
||||
{
|
||||
ip: net.ParseIP("2001:db8::5"),
|
||||
expectIPv6: true,
|
||||
},
|
||||
}
|
||||
for i := range testCases {
|
||||
isIPv6 := IsIPv6(testCases[i].ip)
|
||||
if isIPv6 != testCases[i].expectIPv6 {
|
||||
t.Errorf("[%d] Expect ipv6 %v, got %v", i+1, testCases[i].expectIPv6, isIPv6)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user