From fedd5d62067128528e09c49323883d235a5d1d65 Mon Sep 17 00:00:00 2001 From: Janet Kuo Date: Wed, 6 Jun 2018 11:01:35 -0700 Subject: [PATCH 1/2] Revert "Remove hack in kubectl delete that handles DaemonSet deletion" This reverts commit 10a12ddb347a98f97e9f4c9eb35bc9b8d59384fa. --- pkg/kubectl/cmd/BUILD | 2 ++ pkg/kubectl/cmd/delete.go | 63 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+) diff --git a/pkg/kubectl/cmd/BUILD b/pkg/kubectl/cmd/BUILD index 271c47b5cde..82074478ae1 100644 --- a/pkg/kubectl/cmd/BUILD +++ b/pkg/kubectl/cmd/BUILD @@ -105,6 +105,7 @@ go_library( "//vendor/github.com/renstrom/dedent:go_default_library", "//vendor/github.com/spf13/cobra:go_default_library", "//vendor/github.com/spf13/pflag:go_default_library", + "//vendor/k8s.io/api/apps/v1:go_default_library", "//vendor/k8s.io/api/autoscaling/v1:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", "//vendor/k8s.io/api/policy/v1beta1:go_default_library", @@ -126,6 +127,7 @@ go_library( "//vendor/k8s.io/apimachinery/pkg/util/net:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/strategicpatch:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/uuid:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/validation:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/yaml:go_default_library", diff --git a/pkg/kubectl/cmd/delete.go b/pkg/kubectl/cmd/delete.go index e8ba8ae807a..cbe8abd0a28 100644 --- a/pkg/kubectl/cmd/delete.go +++ b/pkg/kubectl/cmd/delete.go @@ -24,9 +24,15 @@ import ( "github.com/golang/glog" "github.com/spf13/cobra" + appsv1 "k8s.io/api/apps/v1" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/uuid" + "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/dynamic" "k8s.io/kubernetes/pkg/kubectl/cmd/templates" cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" @@ -285,6 +291,16 @@ func (o *DeleteOptions) DeleteResult(r *resource.Result) error { } func (o *DeleteOptions) deleteResource(info *resource.Info, deleteOptions *metav1.DeleteOptions) error { + // TODO: this should be removed as soon as DaemonSet controller properly handles object deletion + // see https://github.com/kubernetes/kubernetes/issues/64313 for details + mapping := info.ResourceMapping() + if mapping.Resource.GroupResource() == (schema.GroupResource{Group: "extensions", Resource: "daemonsets"}) || + mapping.Resource.GroupResource() == (schema.GroupResource{Group: "apps", Resource: "daemonsets"}) { + if err := updateDaemonSet(info.Namespace, info.Name, o.DynamicClient); err != nil { + return err + } + } + if err := resource.NewHelper(info.Client, info.Mapping).DeleteWithOptions(info.Namespace, info.Name, deleteOptions); err != nil { return cmdutil.AddSourceToErr("deleting", info.Source, err) } @@ -293,6 +309,53 @@ func (o *DeleteOptions) deleteResource(info *resource.Info, deleteOptions *metav return nil } +func updateDaemonSet(namespace, name string, dynamicClient dynamic.Interface) error { + dsClient := dynamicClient.Resource(schema.GroupVersionResource{Group: "apps", Version: "v1", Resource: "daemonsets"}).Namespace(namespace) + obj, err := dsClient.Get(name, metav1.GetOptions{}) + if err != nil { + return err + } + ds := &appsv1.DaemonSet{} + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(obj.Object, ds); err != nil { + return err + } + + // We set the nodeSelector to a random label. This label is nearly guaranteed + // to not be set on any node so the DameonSetController will start deleting + // daemon pods. Once it's done deleting the daemon pods, it's safe to delete + // the DaemonSet. + ds.Spec.Template.Spec.NodeSelector = map[string]string{ + string(uuid.NewUUID()): string(uuid.NewUUID()), + } + // force update to avoid version conflict + ds.ResourceVersion = "" + + out, err := runtime.DefaultUnstructuredConverter.ToUnstructured(ds) + if err != nil { + return err + } + if _, err = dsClient.Update(&unstructured.Unstructured{Object: out}); err != nil { + return err + } + + // Wait for the daemon set controller to kill all the daemon pods. + if err := wait.Poll(1*time.Second, 5*time.Minute, func() (bool, error) { + updatedObj, err := dsClient.Get(name, metav1.GetOptions{}) + if err != nil { + return false, nil + } + updatedDS := &appsv1.DaemonSet{} + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(updatedObj.Object, ds); err != nil { + return false, nil + } + + return updatedDS.Status.CurrentNumberScheduled+updatedDS.Status.NumberMisscheduled == 0, nil + }); err != nil { + return err + } + return nil +} + // deletion printing is special because we do not have an object to print. // This mirrors name printer behavior func (o *DeleteOptions) PrintObj(info *resource.Info) { From 710d524b98eb1eb039785e0c8d0694b2b6528d51 Mon Sep 17 00:00:00 2001 From: Janet Kuo Date: Wed, 6 Jun 2018 11:04:41 -0700 Subject: [PATCH 2/2] Add TODO for removing kubectl DaemonSet deletion hack --- pkg/kubectl/cmd/delete.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pkg/kubectl/cmd/delete.go b/pkg/kubectl/cmd/delete.go index cbe8abd0a28..dc0d63ca198 100644 --- a/pkg/kubectl/cmd/delete.go +++ b/pkg/kubectl/cmd/delete.go @@ -291,8 +291,8 @@ func (o *DeleteOptions) DeleteResult(r *resource.Result) error { } func (o *DeleteOptions) deleteResource(info *resource.Info, deleteOptions *metav1.DeleteOptions) error { - // TODO: this should be removed as soon as DaemonSet controller properly handles object deletion - // see https://github.com/kubernetes/kubernetes/issues/64313 for details + // TODO: Remove this in or after 1.12 release. + // Server version >= 1.11 no longer needs this hack. mapping := info.ResourceMapping() if mapping.Resource.GroupResource() == (schema.GroupResource{Group: "extensions", Resource: "daemonsets"}) || mapping.Resource.GroupResource() == (schema.GroupResource{Group: "apps", Resource: "daemonsets"}) { @@ -309,6 +309,8 @@ func (o *DeleteOptions) deleteResource(info *resource.Info, deleteOptions *metav return nil } +// TODO: Remove this in or after 1.12 release. +// Server version >= 1.11 no longer needs this hack. func updateDaemonSet(namespace, name string, dynamicClient dynamic.Interface) error { dsClient := dynamicClient.Resource(schema.GroupVersionResource{Group: "apps", Version: "v1", Resource: "daemonsets"}).Namespace(namespace) obj, err := dsClient.Get(name, metav1.GetOptions{})