From 222988bdda21bd1c41e99e3bdc35239dc3058064 Mon Sep 17 00:00:00 2001 From: Tim Allclair Date: Fri, 22 Feb 2019 11:29:15 -0800 Subject: [PATCH] Migrate to internal node.k8s.io client --- cmd/kubelet/app/BUILD | 1 - cmd/kubelet/app/server.go | 10 --- pkg/kubelet/BUILD | 1 - pkg/kubelet/kubelet.go | 6 +- pkg/kubelet/runtimeclass/BUILD | 7 +-- .../runtimeclass/runtimeclass_manager.go | 32 +++------- .../runtimeclass/runtimeclass_manager_test.go | 1 - pkg/kubelet/runtimeclass/testing/BUILD | 6 +- .../runtimeclass/testing/fake_manager.go | 20 +++--- test/e2e/framework/BUILD | 1 - test/e2e/framework/framework.go | 3 - test/e2e/node/BUILD | 1 - test/e2e/node/runtimeclass.go | 63 +------------------ 13 files changed, 29 insertions(+), 123 deletions(-) diff --git a/cmd/kubelet/app/BUILD b/cmd/kubelet/app/BUILD index 4ead9062441..a837a646d64 100644 --- a/cmd/kubelet/app/BUILD +++ b/cmd/kubelet/app/BUILD @@ -137,7 +137,6 @@ go_library( "//staging/src/k8s.io/cloud-provider:go_default_library", "//staging/src/k8s.io/component-base/cli/flag:go_default_library", "//staging/src/k8s.io/kubelet/config/v1beta1:go_default_library", - "//staging/src/k8s.io/node-api/pkg/client/clientset/versioned:go_default_library", "//vendor/github.com/coreos/go-systemd/daemon:go_default_library", "//vendor/github.com/spf13/cobra:go_default_library", "//vendor/github.com/spf13/pflag:go_default_library", diff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go index d22eb21cb42..69f0e35d0a9 100644 --- a/cmd/kubelet/app/server.go +++ b/cmd/kubelet/app/server.go @@ -95,7 +95,6 @@ import ( "k8s.io/kubernetes/pkg/version" "k8s.io/kubernetes/pkg/version/verflag" "k8s.io/kubernetes/pkg/volume/util/subpath" - nodeapiclientset "k8s.io/node-api/pkg/client/clientset/versioned" "k8s.io/utils/exec" "k8s.io/utils/nsenter" ) @@ -592,15 +591,6 @@ func run(s *options.KubeletServer, kubeDeps *kubelet.Dependencies, stopCh <-chan if err != nil { return fmt.Errorf("failed to initialize kubelet heartbeat client: %v", err) } - - // CRDs are JSON only, and client renegotiation for streaming is not correct as per #67803 - crdClientConfig := restclient.CopyConfig(clientConfig) - crdClientConfig.ContentType = "application/json" - - kubeDeps.NodeAPIClient, err = nodeapiclientset.NewForConfig(crdClientConfig) - if err != nil { - return fmt.Errorf("failed to initialize kubelet node-api client: %v", err) - } } // If the kubelet config controller is available, and dynamic config is enabled, start the config and status sync loops diff --git a/pkg/kubelet/BUILD b/pkg/kubelet/BUILD index 5616e1c6e7b..fb1e04671ba 100644 --- a/pkg/kubelet/BUILD +++ b/pkg/kubelet/BUILD @@ -140,7 +140,6 @@ go_library( "//staging/src/k8s.io/client-go/util/certificate:go_default_library", "//staging/src/k8s.io/client-go/util/flowcontrol:go_default_library", "//staging/src/k8s.io/cloud-provider:go_default_library", - "//staging/src/k8s.io/node-api/pkg/client/clientset/versioned:go_default_library", "//third_party/forked/golang/expansion:go_default_library", "//vendor/github.com/golang/groupcache/lru:go_default_library", "//vendor/github.com/google/cadvisor/events:go_default_library", diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go index edee172ef10..382c57c30af 100644 --- a/pkg/kubelet/kubelet.go +++ b/pkg/kubelet/kubelet.go @@ -113,7 +113,6 @@ import ( "k8s.io/kubernetes/pkg/volume" "k8s.io/kubernetes/pkg/volume/csi" "k8s.io/kubernetes/pkg/volume/util/subpath" - nodeapiclientset "k8s.io/node-api/pkg/client/clientset/versioned" utilexec "k8s.io/utils/exec" "k8s.io/utils/integer" ) @@ -248,7 +247,6 @@ type Dependencies struct { HeartbeatClient clientset.Interface OnHeartbeatFailure func() KubeClient clientset.Interface - NodeAPIClient nodeapiclientset.Interface Mounter mount.Interface OOMAdjuster *oom.OOMAdjuster OSInterface kubecontainer.OSInterface @@ -658,8 +656,8 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration, } klet.runtimeService = runtimeService - if utilfeature.DefaultFeatureGate.Enabled(features.RuntimeClass) && kubeDeps.NodeAPIClient != nil { - klet.runtimeClassManager = runtimeclass.NewManager(kubeDeps.NodeAPIClient) + if utilfeature.DefaultFeatureGate.Enabled(features.RuntimeClass) { + klet.runtimeClassManager = runtimeclass.NewManager(kubeDeps.KubeClient) } runtime, err := kuberuntime.NewKubeGenericRuntimeManager( diff --git a/pkg/kubelet/runtimeclass/BUILD b/pkg/kubelet/runtimeclass/BUILD index 8e59955f3c4..ce13e89088a 100644 --- a/pkg/kubelet/runtimeclass/BUILD +++ b/pkg/kubelet/runtimeclass/BUILD @@ -7,10 +7,9 @@ go_library( visibility = ["//visibility:public"], deps = [ "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", - "//staging/src/k8s.io/node-api/pkg/client/clientset/versioned:go_default_library", - "//staging/src/k8s.io/node-api/pkg/client/informers/externalversions:go_default_library", - "//staging/src/k8s.io/node-api/pkg/client/listers/node/v1alpha1:go_default_library", + "//staging/src/k8s.io/client-go/informers:go_default_library", + "//staging/src/k8s.io/client-go/kubernetes:go_default_library", + "//vendor/k8s.io/client-go/listers/node/v1beta1:go_default_library", ], ) diff --git a/pkg/kubelet/runtimeclass/runtimeclass_manager.go b/pkg/kubelet/runtimeclass/runtimeclass_manager.go index b65d5b9bf13..020275dcfaf 100644 --- a/pkg/kubelet/runtimeclass/runtimeclass_manager.go +++ b/pkg/kubelet/runtimeclass/runtimeclass_manager.go @@ -20,31 +20,23 @@ import ( "fmt" "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/runtime/schema" - nodeapiclient "k8s.io/node-api/pkg/client/clientset/versioned" - nodeapiinformer "k8s.io/node-api/pkg/client/informers/externalversions" - nodev1alpha1 "k8s.io/node-api/pkg/client/listers/node/v1alpha1" -) - -var ( - runtimeClassGVR = schema.GroupVersionResource{ - Group: "node.k8s.io", - Version: "v1alpha1", - Resource: "runtimeclasses", - } + "k8s.io/client-go/informers" + clientset "k8s.io/client-go/kubernetes" + nodev1beta1 "k8s.io/client-go/listers/node/v1beta1" ) // Manager caches RuntimeClass API objects, and provides accessors to the Kubelet. type Manager struct { - informerFactory nodeapiinformer.SharedInformerFactory - lister nodev1alpha1.RuntimeClassLister + informerFactory informers.SharedInformerFactory + lister nodev1beta1.RuntimeClassLister } // NewManager returns a new RuntimeClass Manager. Run must be called before the manager can be used. -func NewManager(client nodeapiclient.Interface) *Manager { +func NewManager(client clientset.Interface) *Manager { const resyncPeriod = 0 - factory := nodeapiinformer.NewSharedInformerFactory(client, resyncPeriod) - lister := factory.Node().V1alpha1().RuntimeClasses().Lister() + + factory := informers.NewSharedInformerFactory(client, resyncPeriod) + lister := factory.Node().V1beta1().RuntimeClasses().Lister() return &Manager{ informerFactory: factory, @@ -82,9 +74,5 @@ func (m *Manager) LookupRuntimeHandler(runtimeClassName *string) (string, error) return "", fmt.Errorf("Failed to lookup RuntimeClass %s: %v", name, err) } - handler := rc.Spec.RuntimeHandler - if handler == nil { - return "", nil - } - return *handler, nil + return rc.Handler, nil } diff --git a/pkg/kubelet/runtimeclass/runtimeclass_manager_test.go b/pkg/kubelet/runtimeclass/runtimeclass_manager_test.go index 004157a1127..676128086db 100644 --- a/pkg/kubelet/runtimeclass/runtimeclass_manager_test.go +++ b/pkg/kubelet/runtimeclass/runtimeclass_manager_test.go @@ -33,7 +33,6 @@ func TestLookupRuntimeHandler(t *testing.T) { expected string expectError bool }{ - {rcn: nil, expected: ""}, {rcn: pointer.StringPtr(""), expected: ""}, {rcn: pointer.StringPtr(rctest.EmptyRuntimeClass), expected: ""}, {rcn: pointer.StringPtr(rctest.SandboxRuntimeClass), expected: "kata-containers"}, diff --git a/pkg/kubelet/runtimeclass/testing/BUILD b/pkg/kubelet/runtimeclass/testing/BUILD index b48519c73a7..18fba5c82d8 100644 --- a/pkg/kubelet/runtimeclass/testing/BUILD +++ b/pkg/kubelet/runtimeclass/testing/BUILD @@ -7,10 +7,10 @@ go_library( visibility = ["//visibility:public"], deps = [ "//pkg/kubelet/runtimeclass:go_default_library", + "//staging/src/k8s.io/api/node/v1beta1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", - "//staging/src/k8s.io/node-api/pkg/apis/node/v1alpha1:go_default_library", - "//staging/src/k8s.io/node-api/pkg/client/clientset/versioned:go_default_library", - "//staging/src/k8s.io/node-api/pkg/client/clientset/versioned/fake:go_default_library", + "//staging/src/k8s.io/client-go/kubernetes:go_default_library", + "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", ], ) diff --git a/pkg/kubelet/runtimeclass/testing/fake_manager.go b/pkg/kubelet/runtimeclass/testing/fake_manager.go index 99b19ab8c07..10e8a445e14 100644 --- a/pkg/kubelet/runtimeclass/testing/fake_manager.go +++ b/pkg/kubelet/runtimeclass/testing/fake_manager.go @@ -17,11 +17,11 @@ limitations under the License. package testing import ( + nodev1beta1 "k8s.io/api/node/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/fake" "k8s.io/kubernetes/pkg/kubelet/runtimeclass" - nodev1alpha1 "k8s.io/node-api/pkg/apis/node/v1alpha1" - nodeapiclient "k8s.io/node-api/pkg/client/clientset/versioned" - nodeapifake "k8s.io/node-api/pkg/client/clientset/versioned/fake" ) const ( @@ -34,10 +34,10 @@ const ( EmptyRuntimeClass = "native" ) -// NewPopulatedClient creates a node-api client for use with the runtimeclass.Manager, +// NewPopulatedClient creates a fake client for use with the runtimeclass.Manager, // and populates it with a few test RuntimeClass objects. -func NewPopulatedClient() nodeapiclient.Interface { - return nodeapifake.NewSimpleClientset( +func NewPopulatedClient() clientset.Interface { + return fake.NewSimpleClientset( NewRuntimeClass(EmptyRuntimeClass, ""), NewRuntimeClass(SandboxRuntimeClass, SandboxRuntimeHandler), ) @@ -57,13 +57,11 @@ func StartManagerSync(m *runtimeclass.Manager) func() { // NewRuntimeClass is a helper to generate a RuntimeClass resource with // the given name & handler. -func NewRuntimeClass(name, handler string) *nodev1alpha1.RuntimeClass { - return &nodev1alpha1.RuntimeClass{ +func NewRuntimeClass(name, handler string) *nodev1beta1.RuntimeClass { + return &nodev1beta1.RuntimeClass{ ObjectMeta: metav1.ObjectMeta{ Name: name, }, - Spec: nodev1alpha1.RuntimeClassSpec{ - RuntimeHandler: &handler, - }, + Handler: handler, } } diff --git a/test/e2e/framework/BUILD b/test/e2e/framework/BUILD index 0c769830a92..945d14fe1ed 100644 --- a/test/e2e/framework/BUILD +++ b/test/e2e/framework/BUILD @@ -127,7 +127,6 @@ go_library( "//staging/src/k8s.io/client-go/util/retry:go_default_library", "//staging/src/k8s.io/component-base/cli/flag:go_default_library", "//staging/src/k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset:go_default_library", - "//staging/src/k8s.io/node-api/pkg/client/clientset/versioned:go_default_library", "//test/e2e/framework/ginkgowrapper:go_default_library", "//test/e2e/framework/metrics:go_default_library", "//test/e2e/framework/testfiles:go_default_library", diff --git a/test/e2e/framework/framework.go b/test/e2e/framework/framework.go index c53c1ae3051..7371fb1605a 100644 --- a/test/e2e/framework/framework.go +++ b/test/e2e/framework/framework.go @@ -51,7 +51,6 @@ import ( "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" "k8s.io/kubernetes/test/e2e/framework/metrics" testutils "k8s.io/kubernetes/test/utils" - nodeapiclient "k8s.io/node-api/pkg/client/clientset/versioned" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -77,7 +76,6 @@ type Framework struct { ClientSet clientset.Interface KubemarkExternalClusterClientSet clientset.Interface APIExtensionsClientSet apiextensionsclient.Interface - NodeAPIClientSet nodeapiclient.Interface InternalClientset *internalclientset.Clientset AggregatorClient *aggregatorclient.Clientset @@ -195,7 +193,6 @@ func (f *Framework) BeforeEach() { // node.k8s.io is based on CRD, which is served only as JSON jsonConfig := config jsonConfig.ContentType = "application/json" - f.NodeAPIClientSet, err = nodeapiclient.NewForConfig(jsonConfig) ExpectNoError(err) // create scales getter, set GroupVersion and NegotiatedSerializer to default values diff --git a/test/e2e/node/BUILD b/test/e2e/node/BUILD index efc51612b61..d4f1b8466b3 100644 --- a/test/e2e/node/BUILD +++ b/test/e2e/node/BUILD @@ -34,7 +34,6 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/fields:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", diff --git a/test/e2e/node/runtimeclass.go b/test/e2e/node/runtimeclass.go index e5fe7f8b324..5e5d4cde127 100644 --- a/test/e2e/node/runtimeclass.go +++ b/test/e2e/node/runtimeclass.go @@ -24,7 +24,6 @@ import ( "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" - "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/kubernetes/pkg/kubelet/events" runtimeclasstest "k8s.io/kubernetes/pkg/kubelet/runtimeclass/testing" @@ -35,16 +34,6 @@ import ( . "github.com/onsi/ginkgo" ) -const runtimeClassCRDName = "runtimeclasses.node.k8s.io" - -var ( - runtimeClassGVR = schema.GroupVersionResource{ - Group: "node.k8s.io", - Version: "v1alpha1", - Resource: "runtimeclasses", - } -) - var _ = SIGDescribe("RuntimeClass [Feature:RuntimeClass]", func() { f := framework.NewDefaultFramework("runtimeclass") @@ -69,7 +58,7 @@ var _ = SIGDescribe("RuntimeClass [Feature:RuntimeClass]", func() { It("should reject a Pod requesting a deleted RuntimeClass", func() { rcName := createRuntimeClass(f, "delete-me", "") - rcClient := f.NodeAPIClientSet.NodeV1alpha1().RuntimeClasses() + rcClient := f.ClientSet.NodeV1beta1().RuntimeClasses() By("Deleting RuntimeClass "+rcName, func() { err := rcClient.Delete(rcName, nil) @@ -92,54 +81,6 @@ var _ = SIGDescribe("RuntimeClass [Feature:RuntimeClass]", func() { expectSandboxFailureEvent(f, pod, fmt.Sprintf("\"%s\" not found", rcName)) }) - It("should recover when the RuntimeClass CRD is deleted [Slow]", func() { - By("Deleting the RuntimeClass CRD", func() { - crds := f.APIExtensionsClientSet.ApiextensionsV1beta1().CustomResourceDefinitions() - runtimeClassCRD, err := crds.Get(runtimeClassCRDName, metav1.GetOptions{}) - framework.ExpectNoError(err, "failed to get RuntimeClass CRD %s", runtimeClassCRDName) - runtimeClassCRDUID := runtimeClassCRD.GetUID() - - err = crds.Delete(runtimeClassCRDName, nil) - framework.ExpectNoError(err, "failed to delete RuntimeClass CRD %s", runtimeClassCRDName) - - By("Waiting for the CRD to disappear") - framework.ExpectNoError(wait.PollImmediate(framework.Poll, time.Minute, func() (bool, error) { - crd, err := crds.Get(runtimeClassCRDName, metav1.GetOptions{}) - if errors.IsNotFound(err) { - return true, nil // done - } - if err != nil { - return true, err // stop wait with error - } - // If the UID changed, that means the addon manager has already recreated it. - return crd.GetUID() != runtimeClassCRDUID, nil - })) - - By("Waiting for the CRD to be recreated") - framework.ExpectNoError(wait.PollImmediate(framework.Poll, 5*time.Minute, func() (bool, error) { - crd, err := crds.Get(runtimeClassCRDName, metav1.GetOptions{}) - if errors.IsNotFound(err) { - return false, nil // still not recreated - } - if err != nil { - return true, err // stop wait with error - } - if crd.GetUID() == runtimeClassCRDUID { - return true, fmt.Errorf("RuntimeClass CRD never deleted") // this shouldn't happen - } - return true, nil - })) - }) - - rcName := createRuntimeClass(f, "valid", "") - pod := createRuntimeClassPod(f, rcName) - - // Before the pod can be run, the RuntimeClass informer must time out, by which time the Kubelet - // will probably be in a backoff state, so the pod can take a long time to start. - framework.ExpectNoError(framework.WaitForPodSuccessInNamespaceSlow( - f.ClientSet, pod.Name, f.Namespace.Name)) - }) - // TODO(tallclair): Test an actual configured non-default runtimeHandler. }) @@ -148,7 +89,7 @@ var _ = SIGDescribe("RuntimeClass [Feature:RuntimeClass]", func() { func createRuntimeClass(f *framework.Framework, name, handler string) string { uniqueName := fmt.Sprintf("%s-%s", f.Namespace.Name, name) rc := runtimeclasstest.NewRuntimeClass(uniqueName, handler) - rc, err := f.NodeAPIClientSet.NodeV1alpha1().RuntimeClasses().Create(rc) + rc, err := f.ClientSet.NodeV1beta1().RuntimeClasses().Create(rc) framework.ExpectNoError(err, "failed to create RuntimeClass resource") return rc.GetName() }