Merge pull request #130561 from danwinship/more-misc-proxy-cleanup

more misc proxy cleanup
This commit is contained in:
Kubernetes Prow Robot 2025-03-07 17:09:44 -08:00 committed by GitHub
commit 1d88598555
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
21 changed files with 57 additions and 104 deletions

View File

@ -175,7 +175,6 @@ func (s *ProxyServer) createProxier(ctx context.Context, config *proxyconfigapi.
ctx,
ipt,
utilsysctl.New(),
exec.New(),
config.SyncPeriod.Duration,
config.MinSyncPeriod.Duration,
config.Linux.MasqueradeAll,
@ -199,7 +198,6 @@ func (s *ProxyServer) createProxier(ctx context.Context, config *proxyconfigapi.
s.PrimaryIPFamily,
iptInterface,
utilsysctl.New(),
exec.New(),
config.SyncPeriod.Duration,
config.MinSyncPeriod.Duration,
config.Linux.MasqueradeAll,
@ -219,8 +217,7 @@ func (s *ProxyServer) createProxier(ctx context.Context, config *proxyconfigapi.
return nil, fmt.Errorf("unable to create proxier: %v", err)
}
} else if config.Mode == proxyconfigapi.ProxyModeIPVS {
execer := exec.New()
ipsetInterface := utilipset.New(execer)
ipsetInterface := utilipset.New()
ipvsInterface := utilipvs.New()
if err := ipvs.CanUseIPVSProxier(ctx, ipvsInterface, ipsetInterface, config.IPVS.Scheduler); err != nil {
return nil, fmt.Errorf("can't use the IPVS proxier: %v", err)
@ -235,7 +232,6 @@ func (s *ProxyServer) createProxier(ctx context.Context, config *proxyconfigapi.
ipvsInterface,
ipsetInterface,
utilsysctl.New(),
execer,
config.SyncPeriod.Duration,
config.MinSyncPeriod.Duration,
config.IPVS.ExcludeCIDRs,
@ -263,7 +259,6 @@ func (s *ProxyServer) createProxier(ctx context.Context, config *proxyconfigapi.
ipvsInterface,
ipsetInterface,
utilsysctl.New(),
execer,
config.SyncPeriod.Duration,
config.MinSyncPeriod.Duration,
config.IPVS.ExcludeCIDRs,
@ -515,8 +510,7 @@ func platformCleanup(ctx context.Context, mode proxyconfigapi.ProxyMode, cleanup
// Clean up iptables and ipvs rules if switching to nftables, or if cleanupAndExit
if !isIPTablesBased(mode) || cleanupAndExit {
ipts, _ := getIPTables(v1.IPFamilyUnknown)
execer := exec.New()
ipsetInterface := utilipset.New(execer)
ipsetInterface := utilipset.New()
ipvsInterface := utilipvs.New()
for _, ipt := range ipts {

View File

@ -19,10 +19,6 @@ limitations under the License.
package iptables
//
// NOTE: this needs to be tested in e2e since it uses iptables for everything.
//
import (
"bytes"
"context"
@ -54,7 +50,6 @@ import (
"k8s.io/kubernetes/pkg/proxy/util/nfacct"
"k8s.io/kubernetes/pkg/util/async"
utiliptables "k8s.io/kubernetes/pkg/util/iptables"
utilexec "k8s.io/utils/exec"
)
const (
@ -101,7 +96,6 @@ func NewDualStackProxier(
ctx context.Context,
ipt [2]utiliptables.Interface,
sysctl utilsysctl.Interface,
exec utilexec.Interface,
syncPeriod time.Duration,
minSyncPeriod time.Duration,
masqueradeAll bool,
@ -117,7 +111,7 @@ func NewDualStackProxier(
) (proxy.Provider, error) {
// Create an ipv4 instance of the single-stack proxier
ipv4Proxier, err := NewProxier(ctx, v1.IPv4Protocol, ipt[0], sysctl,
exec, syncPeriod, minSyncPeriod, masqueradeAll, localhostNodePorts, masqueradeBit,
syncPeriod, minSyncPeriod, masqueradeAll, localhostNodePorts, masqueradeBit,
localDetectors[v1.IPv4Protocol], hostname, nodeIPs[v1.IPv4Protocol],
recorder, healthzServer, nodePortAddresses, initOnly)
if err != nil {
@ -125,7 +119,7 @@ func NewDualStackProxier(
}
ipv6Proxier, err := NewProxier(ctx, v1.IPv6Protocol, ipt[1], sysctl,
exec, syncPeriod, minSyncPeriod, masqueradeAll, false, masqueradeBit,
syncPeriod, minSyncPeriod, masqueradeAll, false, masqueradeBit,
localDetectors[v1.IPv6Protocol], hostname, nodeIPs[v1.IPv6Protocol],
recorder, healthzServer, nodePortAddresses, initOnly)
if err != nil {
@ -137,8 +131,7 @@ func NewDualStackProxier(
return metaproxier.NewMetaProxier(ipv4Proxier, ipv6Proxier), nil
}
// Proxier is an iptables based proxy for connections between a localhost:lport
// and services that provide the actual backends.
// Proxier is an iptables-based proxy
type Proxier struct {
// ipFamily defines the IP family which this proxier is tracking.
ipFamily v1.IPFamily
@ -175,7 +168,6 @@ type Proxier struct {
localDetector proxyutil.LocalTrafficDetector
hostname string
nodeIP net.IP
recorder events.EventRecorder
serviceHealthServer healthcheck.ServiceHealthServer
healthzServer *healthcheck.ProxyHealthServer
@ -221,16 +213,11 @@ type Proxier struct {
// Proxier implements proxy.Provider
var _ proxy.Provider = &Proxier{}
// NewProxier returns a new Proxier given an iptables Interface instance.
// Because of the iptables logic, it is assumed that there is only a single Proxier active on a machine.
// An error will be returned if iptables fails to update or acquire the initial lock.
// Once a proxier is created, it will keep iptables up to date in the background and
// will not terminate if a particular iptables call fails.
// NewProxier returns a new single-stack IPTables proxier.
func NewProxier(ctx context.Context,
ipFamily v1.IPFamily,
ipt utiliptables.Interface,
sysctl utilsysctl.Interface,
exec utilexec.Interface,
syncPeriod time.Duration,
minSyncPeriod time.Duration,
masqueradeAll bool,
@ -300,7 +287,6 @@ func NewProxier(ctx context.Context,
localDetector: localDetector,
hostname: hostname,
nodeIP: nodeIP,
recorder: recorder,
serviceHealthServer: serviceHealthServer,
healthzServer: healthzServer,
precomputedProbabilities: make([]string, 0, 1001),

View File

@ -35,6 +35,7 @@ import (
"github.com/google/go-cmp/cmp"
"github.com/lithammer/dedent"
"github.com/stretchr/testify/assert"
v1 "k8s.io/api/core/v1"
discovery "k8s.io/api/discovery/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

View File

@ -24,10 +24,9 @@ import (
"reflect"
"testing"
netutils "k8s.io/utils/net"
utilipvs "k8s.io/kubernetes/pkg/proxy/ipvs/util"
utilipvstest "k8s.io/kubernetes/pkg/proxy/ipvs/util/testing"
netutils "k8s.io/utils/net"
)
func Test_GracefulDeleteRS(t *testing.T) {

View File

@ -20,14 +20,13 @@ limitations under the License.
package ipvs
import (
"k8s.io/apimachinery/pkg/util/sets"
utilversion "k8s.io/apimachinery/pkg/util/version"
utilipset "k8s.io/kubernetes/pkg/proxy/ipvs/ipset"
"fmt"
"strings"
"k8s.io/apimachinery/pkg/util/sets"
utilversion "k8s.io/apimachinery/pkg/util/version"
"k8s.io/klog/v2"
utilipset "k8s.io/kubernetes/pkg/proxy/ipvs/ipset"
)
const (

View File

@ -278,9 +278,9 @@ type runner struct {
}
// New returns a new Interface which will exec ipset.
func New(exec utilexec.Interface) Interface {
func New() Interface {
return &runner{
exec: exec,
exec: utilexec.New(),
}
}

View File

@ -28,6 +28,10 @@ import (
fakeexec "k8s.io/utils/exec/testing"
)
func newInternal(fexec *fakeexec.FakeExec) Interface {
return &runner{fexec}
}
func TestCheckIPSetVersion(t *testing.T) {
testCases := []struct {
vstring string
@ -83,7 +87,7 @@ func TestFlushSet(t *testing.T) {
func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) },
},
}
runner := New(fexec)
runner := newInternal(fexec)
// Success.
err := runner.FlushSet("FOOBAR")
if err != nil {
@ -119,7 +123,7 @@ func TestDestroySet(t *testing.T) {
func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) },
},
}
runner := New(fexec)
runner := newInternal(fexec)
// Success
err := runner.DestroySet("FOOBAR")
if err != nil {
@ -153,7 +157,7 @@ func TestDestroyAllSets(t *testing.T) {
func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) },
},
}
runner := New(fexec)
runner := newInternal(fexec)
// Success
err := runner.DestroyAllSets()
if err != nil {
@ -198,7 +202,7 @@ func TestCreateSet(t *testing.T) {
func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) },
},
}
runner := New(fexec)
runner := newInternal(fexec)
// Create with ignoreExistErr = false, expect success
err := runner.CreateSet(&testSet, false)
if err != nil {
@ -388,7 +392,7 @@ func TestAddEntry(t *testing.T) {
func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) },
},
}
runner := New(fexec)
runner := newInternal(fexec)
// Create with ignoreExistErr = false, expect success
err := runner.AddEntry(testCases[i].entry.String(), testCases[i].set, false)
if err != nil {
@ -437,7 +441,7 @@ func TestDelEntry(t *testing.T) {
func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) },
},
}
runner := New(fexec)
runner := newInternal(fexec)
err := runner.DelEntry(testCases[i].entry.String(), testCases[i].set.Name)
if err != nil {
@ -482,7 +486,7 @@ func TestTestEntry(t *testing.T) {
func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) },
},
}
runner := New(fexec)
runner := newInternal(fexec)
// Success
ok, err := runner.TestEntry(testEntry.String(), setName)
if err != nil {
@ -530,7 +534,7 @@ func TestTestEntryIPv6(t *testing.T) {
func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) },
},
}
runner := New(fexec)
runner := newInternal(fexec)
// Success
ok, err := runner.TestEntry(testEntry.String(), setName)
if err != nil {
@ -604,7 +608,7 @@ Members:
},
},
}
runner := New(fexec)
runner := newInternal(fexec)
// Success
entries, err := runner.ListEntries("foobar")
if err != nil {
@ -643,7 +647,7 @@ baz`
func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) },
},
}
runner := New(fexec)
runner := newInternal(fexec)
// Success
list, err := runner.ListSets()
if err != nil {

View File

@ -23,13 +23,13 @@ import (
"fmt"
"net"
"github.com/vishvananda/netlink"
"golang.org/x/sys/unix"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/klog/v2"
proxyutil "k8s.io/kubernetes/pkg/proxy/util"
netutils "k8s.io/utils/net"
"github.com/vishvananda/netlink"
"golang.org/x/sys/unix"
)
type netlinkHandle struct {

View File

@ -33,10 +33,6 @@ import (
"sync/atomic"
"time"
"k8s.io/klog/v2"
utilexec "k8s.io/utils/exec"
netutils "k8s.io/utils/net"
v1 "k8s.io/api/core/v1"
discovery "k8s.io/api/discovery/v1"
"k8s.io/apimachinery/pkg/types"
@ -45,6 +41,7 @@ import (
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/tools/events"
utilsysctl "k8s.io/component-helpers/node/util/sysctl"
"k8s.io/klog/v2"
"k8s.io/kubernetes/pkg/proxy"
"k8s.io/kubernetes/pkg/proxy/conntrack"
"k8s.io/kubernetes/pkg/proxy/healthcheck"
@ -56,6 +53,7 @@ import (
"k8s.io/kubernetes/pkg/util/async"
utiliptables "k8s.io/kubernetes/pkg/util/iptables"
utilkernel "k8s.io/kubernetes/pkg/util/kernel"
netutils "k8s.io/utils/net"
)
const (
@ -116,7 +114,6 @@ func NewDualStackProxier(
ipvs utilipvs.Interface,
ipset utilipset.Interface,
sysctl utilsysctl.Interface,
exec utilexec.Interface,
syncPeriod time.Duration,
minSyncPeriod time.Duration,
excludeCIDRs []string,
@ -137,7 +134,7 @@ func NewDualStackProxier(
) (proxy.Provider, error) {
// Create an ipv4 instance of the single-stack proxier
ipv4Proxier, err := NewProxier(ctx, v1.IPv4Protocol, ipt[0], ipvs, ipset, sysctl,
exec, syncPeriod, minSyncPeriod, filterCIDRs(false, excludeCIDRs), strictARP,
syncPeriod, minSyncPeriod, filterCIDRs(false, excludeCIDRs), strictARP,
tcpTimeout, tcpFinTimeout, udpTimeout, masqueradeAll, masqueradeBit,
localDetectors[v1.IPv4Protocol], hostname, nodeIPs[v1.IPv4Protocol], recorder,
healthzServer, scheduler, nodePortAddresses, initOnly)
@ -146,7 +143,7 @@ func NewDualStackProxier(
}
ipv6Proxier, err := NewProxier(ctx, v1.IPv6Protocol, ipt[1], ipvs, ipset, sysctl,
exec, syncPeriod, minSyncPeriod, filterCIDRs(true, excludeCIDRs), strictARP,
syncPeriod, minSyncPeriod, filterCIDRs(true, excludeCIDRs), strictARP,
tcpTimeout, tcpFinTimeout, udpTimeout, masqueradeAll, masqueradeBit,
localDetectors[v1.IPv6Protocol], hostname, nodeIPs[v1.IPv6Protocol], recorder,
healthzServer, scheduler, nodePortAddresses, initOnly)
@ -162,8 +159,7 @@ func NewDualStackProxier(
return metaproxier.NewMetaProxier(ipv4Proxier, ipv6Proxier), nil
}
// Proxier is an ipvs based proxy for connections between a localhost:lport
// and services that provide the actual backends.
// Proxier is an ipvs-based proxy
type Proxier struct {
// the ipfamily on which this proxy is operating on.
ipFamily v1.IPFamily
@ -198,8 +194,7 @@ type Proxier struct {
minSyncPeriod time.Duration
// Values are CIDR's to exclude when cleaning up IPVS rules.
excludeCIDRs []*net.IPNet
// Set to true to set sysctls arp_ignore and arp_announce
strictARP bool
iptables utiliptables.Interface
ipvs utilipvs.Interface
ipset utilipset.Interface
@ -209,7 +204,6 @@ type Proxier struct {
localDetector proxyutil.LocalTrafficDetector
hostname string
nodeIP net.IP
recorder events.EventRecorder
serviceHealthServer healthcheck.ServiceHealthServer
healthzServer *healthcheck.ProxyHealthServer
@ -259,11 +253,7 @@ type Proxier struct {
// Proxier implements proxy.Provider
var _ proxy.Provider = &Proxier{}
// NewProxier returns a new Proxier given an iptables and ipvs Interface instance.
// Because of the iptables and ipvs logic, it is assumed that there is only a single Proxier active on a machine.
// An error will be returned if it fails to update or acquire the initial lock.
// Once a proxier is created, it will keep iptables and ipvs rules up to date in the background and
// will not terminate if a particular iptables or ipvs call fails.
// NewProxier returns a new single-stack IPVS proxier.
func NewProxier(
ctx context.Context,
ipFamily v1.IPFamily,
@ -271,7 +261,6 @@ func NewProxier(
ipvs utilipvs.Interface,
ipset utilipset.Interface,
sysctl utilsysctl.Interface,
exec utilexec.Interface,
syncPeriod time.Duration,
minSyncPeriod time.Duration,
excludeCIDRs []string,
@ -389,7 +378,6 @@ func NewProxier(
localDetector: localDetector,
hostname: hostname,
nodeIP: nodeIP,
recorder: recorder,
serviceHealthServer: serviceHealthServer,
healthzServer: healthzServer,
ipvs: ipvs,

View File

@ -31,6 +31,7 @@ import (
"time"
"github.com/stretchr/testify/assert"
v1 "k8s.io/api/core/v1"
discovery "k8s.io/api/discovery/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@ -148,7 +149,6 @@ func NewFakeProxier(ctx context.Context, ipt utiliptables.Interface, ipvs utilip
ipvs: ipvs,
ipset: ipset,
conntrack: conntrack.NewFake(),
strictARP: false,
localDetector: proxyutil.NewNoOpLocalDetector(),
hostname: testHostname,
serviceHealthServer: healthcheck.NewFakeServiceHealthServer(),

View File

@ -21,9 +21,9 @@ package testing
import (
"fmt"
"k8s.io/utils/net"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/utils/net"
)
// FakeNetlinkHandle mock implementation of proxy NetlinkHandle

View File

@ -28,8 +28,8 @@ import (
"time"
libipvs "github.com/moby/ipvs"
"golang.org/x/sys/unix"
"k8s.io/klog/v2"
)

View File

@ -24,11 +24,10 @@ import (
"reflect"
"testing"
netutils "k8s.io/utils/net"
libipvs "github.com/moby/ipvs"
"golang.org/x/sys/unix"
netutils "k8s.io/utils/net"
)
func Test_toVirtualServer(t *testing.T) {

View File

@ -21,7 +21,6 @@ import (
discovery "k8s.io/api/discovery/v1"
"k8s.io/klog/v2"
"k8s.io/kubernetes/pkg/proxy"
"k8s.io/kubernetes/pkg/proxy/config"
)
type metaProxier struct {
@ -29,8 +28,6 @@ type metaProxier struct {
ipv4Proxier proxy.Provider
// actual, wrapped
ipv6Proxier proxy.Provider
// TODO(imroc): implement node handler for meta proxier.
config.NoopNodeHandler
}
// NewMetaProxier returns a dual-stack "meta-proxier". Proxier API

View File

@ -19,16 +19,11 @@ limitations under the License.
package nftables
//
// NOTE: this needs to be tested in e2e since it uses nftables for everything.
//
import (
"context"
"crypto/sha256"
"encoding/base32"
"fmt"
"golang.org/x/time/rate"
"net"
"os"
"os/exec"
@ -39,7 +34,9 @@ import (
"sync/atomic"
"time"
v1 "k8s.io/api/core/v1"
"golang.org/x/time/rate"
"k8s.io/api/core/v1"
discovery "k8s.io/api/discovery/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/sets"
@ -142,7 +139,7 @@ func NewDualStackProxier(
return metaproxier.NewMetaProxier(ipv4Proxier, ipv6Proxier), nil
}
// Proxier is an nftables based proxy
// Proxier is an nftables-based proxy
type Proxier struct {
// ipFamily defines the IP family which this proxier is tracking.
ipFamily v1.IPFamily
@ -179,7 +176,6 @@ type Proxier struct {
localDetector proxyutil.LocalTrafficDetector
hostname string
nodeIP net.IP
recorder events.EventRecorder
serviceHealthServer healthcheck.ServiceHealthServer
healthzServer *healthcheck.ProxyHealthServer
@ -211,9 +207,7 @@ type Proxier struct {
// Proxier implements proxy.Provider
var _ proxy.Provider = &Proxier{}
// NewProxier returns a new nftables Proxier. Once a proxier is created, it will keep
// nftables up to date in the background and will not terminate if a particular nftables
// call fails.
// NewProxier returns a new single-stack NFTables proxier.
func NewProxier(ctx context.Context,
ipFamily v1.IPFamily,
syncPeriod time.Duration,
@ -264,7 +258,6 @@ func NewProxier(ctx context.Context,
localDetector: localDetector,
hostname: hostname,
nodeIP: nodeIP,
recorder: recorder,
serviceHealthServer: serviceHealthServer,
healthzServer: healthzServer,
nodePortAddresses: nodePortAddresses,

View File

@ -21,7 +21,6 @@ package nftables
import (
"fmt"
"golang.org/x/time/rate"
"net"
"reflect"
"testing"
@ -29,6 +28,7 @@ import (
"github.com/lithammer/dedent"
"github.com/stretchr/testify/assert"
"golang.org/x/time/rate"
v1 "k8s.io/api/core/v1"
discovery "k8s.io/api/discovery/v1"

View File

@ -21,6 +21,7 @@ package winkernel
import (
"github.com/Microsoft/hnslib/hcn"
"k8s.io/klog/v2"
)

View File

@ -23,11 +23,11 @@ import (
"crypto/sha1"
"encoding/json"
"fmt"
"strings"
"github.com/Microsoft/hnslib/hcn"
"k8s.io/klog/v2"
"strings"
"k8s.io/klog/v2"
)
type HostNetworkService interface {

View File

@ -21,14 +21,12 @@ package winkernel
import (
"encoding/json"
"github.com/Microsoft/hnslib/hcn"
"github.com/stretchr/testify/assert"
"strings"
"testing"
"github.com/Microsoft/hnslib/hcn"
"github.com/google/go-cmp/cmp"
"github.com/stretchr/testify/assert"
)
const (

View File

@ -31,6 +31,7 @@ import (
"github.com/Microsoft/hnslib"
"github.com/Microsoft/hnslib/hcn"
v1 "k8s.io/api/core/v1"
discovery "k8s.io/api/discovery/v1"
"k8s.io/apimachinery/pkg/util/intstr"
@ -622,8 +623,7 @@ func (network hnsNetworkInfo) findRemoteSubnetProviderAddress(ip string) string
type endPointsReferenceCountMap map[string]*uint16
// Proxier is an hns based proxy for connections between a localhost:lport
// and services that provide the actual backends.
// Proxier is an HNS-based proxy
type Proxier struct {
// ipFamily defines the IP family which this proxier is tracking.
ipFamily v1.IPFamily
@ -650,7 +650,6 @@ type Proxier struct {
// These are effectively const and do not need the mutex to be held.
hostname string
nodeIP net.IP
recorder events.EventRecorder
serviceHealthServer healthcheck.ServiceHealthServer
healthzServer *healthcheck.ProxyHealthServer
@ -701,7 +700,7 @@ type closeable interface {
// Proxier implements proxy.Provider
var _ proxy.Provider = &Proxier{}
// NewProxier returns a new Proxier
// NewProxier returns a new single-stack winkernel proxier.
func NewProxier(
ipFamily v1.IPFamily,
syncPeriod time.Duration,
@ -713,11 +712,6 @@ func NewProxier(
healthzBindAddress string,
config config.KubeProxyWinkernelConfiguration,
) (*Proxier, error) {
if nodeIP == nil {
klog.InfoS("Invalid nodeIP, initializing kube-proxy with 127.0.0.1 as nodeIP")
nodeIP = netutils.ParseIPSloppy("127.0.0.1")
}
// windows listens to all node addresses
nodePortAddresses := proxyutil.NewNodePortAddresses(ipFamily, nil)
serviceHealthServer := healthcheck.NewServiceHealthServer(hostname, recorder, nodePortAddresses, healthzServer)
@ -812,7 +806,6 @@ func NewProxier(
endpointsMap: make(proxy.EndpointsMap),
hostname: hostname,
nodeIP: nodeIP,
recorder: recorder,
serviceHealthServer: serviceHealthServer,
healthzServer: healthzServer,
hns: hns,

View File

@ -28,6 +28,7 @@ import (
"time"
"github.com/Microsoft/hnslib/hcn"
v1 "k8s.io/api/core/v1"
discovery "k8s.io/api/discovery/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"