Merge pull request #99291 from zshihang/master

remove secret-based sa token client builder
This commit is contained in:
Kubernetes Prow Robot 2021-02-22 07:29:43 -08:00 committed by GitHub
commit 7cea81ce34
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 56 additions and 275 deletions

View File

@ -108,6 +108,7 @@ go_library(
"//pkg/volume/vsphere_volume:go_default_library",
"//staging/src/k8s.io/api/core/v1:go_default_library",
"//staging/src/k8s.io/api/discovery/v1beta1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",

View File

@ -30,7 +30,7 @@ import (
"time"
"github.com/spf13/cobra"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime/schema"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/sets"
@ -43,7 +43,6 @@ import (
utilfeature "k8s.io/apiserver/pkg/util/feature"
cacheddiscovery "k8s.io/client-go/discovery/cached"
"k8s.io/client-go/informers"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/client-go/metadata"
"k8s.io/client-go/metadata/metadatainformer"
restclient "k8s.io/client-go/rest"
@ -65,7 +64,6 @@ import (
"k8s.io/klog/v2"
"k8s.io/kubernetes/cmd/kube-controller-manager/app/config"
"k8s.io/kubernetes/cmd/kube-controller-manager/app/options"
"k8s.io/kubernetes/pkg/controller"
kubectrlmgrconfig "k8s.io/kubernetes/pkg/controller/apis/config"
serviceaccountcontroller "k8s.io/kubernetes/pkg/controller/serviceaccount"
"k8s.io/kubernetes/pkg/serviceaccount"
@ -221,22 +219,10 @@ func Run(c *config.CompletedConfig, stopCh <-chan struct{}) error {
klog.Warningf("--use-service-account-credentials was specified without providing a --service-account-private-key-file")
}
if shouldTurnOnDynamicClient(c.Client) {
klog.V(1).Infof("using dynamic client builder")
//Dynamic builder will use TokenRequest feature and refresh service account token periodically
clientBuilder = controller.NewDynamicClientBuilder(
restclient.AnonymousClientConfig(c.Kubeconfig),
c.Client.CoreV1(),
"kube-system")
} else {
klog.V(1).Infof("using legacy client builder")
clientBuilder = clientbuilder.SAControllerClientBuilder{
ClientConfig: restclient.AnonymousClientConfig(c.Kubeconfig),
CoreClient: c.Client.CoreV1(),
AuthenticationClient: c.Client.AuthenticationV1(),
Namespace: "kube-system",
}
}
clientBuilder = clientbuilder.NewDynamicClientBuilder(
restclient.AnonymousClientConfig(c.Kubeconfig),
c.Client.CoreV1(),
metav1.NamespaceSystem)
} else {
clientBuilder = rootClientBuilder
}
@ -623,21 +609,3 @@ func readCA(file string) ([]byte, error) {
return rootCA, err
}
func shouldTurnOnDynamicClient(client clientset.Interface) bool {
apiResourceList, err := client.Discovery().ServerResourcesForGroupVersion(v1.SchemeGroupVersion.String())
if err != nil {
klog.Warningf("fetch api resource lists failed, use legacy client builder: %v", err)
return false
}
for _, resource := range apiResourceList.APIResources {
if resource.Name == "serviceaccounts/token" &&
resource.Group == "authentication.k8s.io" &&
sets.NewString(resource.Verbs...).Has("create") {
return true
}
}
return false
}

View File

@ -3,7 +3,6 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
go_library(
name = "go_default_library",
srcs = [
"client_builder_dynamic.go",
"controller_ref_manager.go",
"controller_utils.go",
"doc.go",
@ -18,7 +17,6 @@ go_library(
"//pkg/util/hash:go_default_library",
"//pkg/util/taints:go_default_library",
"//staging/src/k8s.io/api/apps/v1:go_default_library",
"//staging/src/k8s.io/api/authentication/v1:go_default_library",
"//staging/src/k8s.io/api/core/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/api/meta:go_default_library",
@ -33,20 +31,13 @@ go_library(
"//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/strategicpatch:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/authentication/serviceaccount:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library",
"//staging/src/k8s.io/client-go/rest:go_default_library",
"//staging/src/k8s.io/client-go/tools/cache:go_default_library",
"//staging/src/k8s.io/client-go/tools/record:go_default_library",
"//staging/src/k8s.io/client-go/transport:go_default_library",
"//staging/src/k8s.io/client-go/util/retry:go_default_library",
"//staging/src/k8s.io/controller-manager/pkg/clientbuilder:go_default_library",
"//vendor/github.com/golang/groupcache/lru:go_default_library",
"//vendor/golang.org/x/oauth2:go_default_library",
"//vendor/k8s.io/klog/v2:go_default_library",
"//vendor/k8s.io/utils/integer:go_default_library",
"//vendor/k8s.io/utils/pointer:go_default_library",
],
)

View File

@ -40,7 +40,6 @@ import (
"k8s.io/apimachinery/pkg/util/strategicpatch"
"k8s.io/apimachinery/pkg/util/wait"
clientset "k8s.io/client-go/kubernetes"
v1core "k8s.io/client-go/kubernetes/typed/core/v1"
"k8s.io/client-go/tools/cache"
"k8s.io/client-go/tools/record"
clientretry "k8s.io/client-go/util/retry"
@ -1183,29 +1182,3 @@ func AddOrUpdateLabelsOnNode(kubeClient clientset.Interface, nodeName string, la
return nil
})
}
func getOrCreateServiceAccount(coreClient v1core.CoreV1Interface, namespace, name string) (*v1.ServiceAccount, error) {
sa, err := coreClient.ServiceAccounts(namespace).Get(context.TODO(), name, metav1.GetOptions{})
if err == nil {
return sa, nil
}
if !apierrors.IsNotFound(err) {
return nil, err
}
// Create the namespace if we can't verify it exists.
// Tolerate errors, since we don't know whether this component has namespace creation permissions.
if _, err := coreClient.Namespaces().Get(context.TODO(), namespace, metav1.GetOptions{}); apierrors.IsNotFound(err) {
if _, err = coreClient.Namespaces().Create(context.TODO(), &v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: namespace}}, metav1.CreateOptions{}); err != nil && !apierrors.IsAlreadyExists(err) {
klog.Warningf("create non-exist namespace %s failed:%v", namespace, err)
}
}
// Create the service account
sa, err = coreClient.ServiceAccounts(namespace).Create(context.TODO(), &v1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: name}}, metav1.CreateOptions{})
if apierrors.IsAlreadyExists(err) {
// If we're racing to init and someone else already created it, re-fetch
return coreClient.ServiceAccounts(namespace).Get(context.TODO(), name, metav1.GetOptions{})
}
return sa, err
}

View File

@ -194,12 +194,10 @@ func (o *CloudControllerManagerOptions) ApplyTo(c *config.Config, userAgent stri
ClientConfig: c.Kubeconfig,
}
if c.ComponentConfig.KubeCloudShared.UseServiceAccountCredentials {
c.ClientBuilder = clientbuilder.SAControllerClientBuilder{
ClientConfig: restclient.AnonymousClientConfig(c.Kubeconfig),
CoreClient: c.Client.CoreV1(),
AuthenticationClient: c.Client.AuthenticationV1(),
Namespace: metav1.NamespaceSystem,
}
c.ClientBuilder = clientbuilder.NewDynamicClientBuilder(
restclient.AnonymousClientConfig(c.Kubeconfig),
c.Client.CoreV1(),
metav1.NamespaceSystem)
} else {
c.ClientBuilder = rootClientBuilder
}

View File

@ -7,12 +7,14 @@ go 1.15
require (
github.com/spf13/pflag v1.0.5
github.com/stretchr/testify v1.6.1
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
k8s.io/api v0.0.0
k8s.io/apimachinery v0.0.0
k8s.io/apiserver v0.0.0
k8s.io/client-go v0.0.0
k8s.io/component-base v0.0.0
k8s.io/klog/v2 v2.5.0
k8s.io/utils v0.0.0-20201110183641-67b214c5f920
)
replace (

View File

@ -2,7 +2,10 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library")
go_library(
name = "go_default_library",
srcs = ["client_builder.go"],
srcs = [
"client_builder.go",
"client_builder_dynamic.go",
],
importmap = "k8s.io/kubernetes/vendor/k8s.io/controller-manager/pkg/clientbuilder",
importpath = "k8s.io/controller-manager/pkg/clientbuilder",
visibility = ["//visibility:public"],
@ -11,18 +14,16 @@ go_library(
"//staging/src/k8s.io/api/core/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/fields:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/clock:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/authentication/serviceaccount:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes/scheme:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes/typed/authentication/v1:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library",
"//staging/src/k8s.io/client-go/rest:go_default_library",
"//staging/src/k8s.io/client-go/tools/cache:go_default_library",
"//staging/src/k8s.io/client-go/tools/watch:go_default_library",
"//staging/src/k8s.io/client-go/transport:go_default_library",
"//vendor/golang.org/x/oauth2:go_default_library",
"//vendor/k8s.io/klog/v2:go_default_library",
"//vendor/k8s.io/utils/pointer:go_default_library",
],
)

View File

@ -17,33 +17,11 @@ limitations under the License.
package clientbuilder
import (
"context"
"fmt"
"time"
v1authenticationapi "k8s.io/api/authentication/v1"
v1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/fields"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/watch"
apiserverserviceaccount "k8s.io/apiserver/pkg/authentication/serviceaccount"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/client-go/kubernetes/scheme"
v1authentication "k8s.io/client-go/kubernetes/typed/authentication/v1"
v1core "k8s.io/client-go/kubernetes/typed/core/v1"
restclient "k8s.io/client-go/rest"
"k8s.io/client-go/tools/cache"
watchtools "k8s.io/client-go/tools/watch"
"k8s.io/klog/v2"
)
const (
// SecretTypeField is copied from pkg/apis/cores/field_constants.go
SecretTypeField = "type"
)
// ControllerClientBuilder allows you to get clients and configs for controllers
// Please note a copy also exists in staging/src/k8s.io/cloud-provider/cloud.go
// TODO: Extract this into a separate controller utilities repo (issues/68947)
@ -94,162 +72,3 @@ func (b SimpleControllerClientBuilder) ClientOrDie(name string) clientset.Interf
}
return client
}
// SAControllerClientBuilder is a ControllerClientBuilder that returns clients identifying as
// service accounts
type SAControllerClientBuilder struct {
// ClientConfig is a skeleton config to clone and use as the basis for each controller client
ClientConfig *restclient.Config
// CoreClient is used to provision service accounts if needed and watch for their associated tokens
// to construct a controller client
CoreClient v1core.CoreV1Interface
// AuthenticationClient is used to check API tokens to make sure they are valid before
// building a controller client from them
AuthenticationClient v1authentication.AuthenticationV1Interface
// Namespace is the namespace used to host the service accounts that will back the
// controllers. It must be highly privileged namespace which normal users cannot inspect.
Namespace string
}
// Config returns a complete clientConfig for constructing clients. This is separate in anticipation of composition
// which means that not all clientsets are known here
func (b SAControllerClientBuilder) Config(name string) (*restclient.Config, error) {
sa, err := apiserverserviceaccount.GetOrCreateServiceAccount(b.CoreClient, b.Namespace, name)
if err != nil {
return nil, err
}
var clientConfig *restclient.Config
fieldSelector := fields.SelectorFromSet(map[string]string{
SecretTypeField: string(v1.SecretTypeServiceAccountToken),
}).String()
lw := &cache.ListWatch{
ListFunc: func(options metav1.ListOptions) (runtime.Object, error) {
options.FieldSelector = fieldSelector
return b.CoreClient.Secrets(b.Namespace).List(context.TODO(), options)
},
WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
options.FieldSelector = fieldSelector
return b.CoreClient.Secrets(b.Namespace).Watch(context.TODO(), options)
},
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
_, err = watchtools.UntilWithSync(ctx, lw, &v1.Secret{}, nil,
func(event watch.Event) (bool, error) {
switch event.Type {
case watch.Deleted:
return false, nil
case watch.Error:
return false, fmt.Errorf("error watching")
case watch.Added, watch.Modified:
secret, ok := event.Object.(*v1.Secret)
if !ok {
return false, fmt.Errorf("unexpected object type: %T", event.Object)
}
if !apiserverserviceaccount.IsServiceAccountToken(secret, sa) {
return false, nil
}
if len(secret.Data[v1.ServiceAccountTokenKey]) == 0 {
return false, nil
}
validConfig, valid, err := b.getAuthenticatedConfig(sa, string(secret.Data[v1.ServiceAccountTokenKey]))
if err != nil {
klog.Warningf("error validating API token for %s/%s in secret %s: %v", sa.Namespace, sa.Name, secret.Name, err)
// continue watching for good tokens
return false, nil
}
if !valid {
klog.Warningf("secret %s contained an invalid API token for %s/%s", secret.Name, sa.Namespace, sa.Name)
// try to delete the secret containing the invalid token
if err := b.CoreClient.Secrets(secret.Namespace).Delete(context.TODO(), secret.Name, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) {
klog.Warningf("error deleting secret %s containing invalid API token for %s/%s: %v", secret.Name, sa.Namespace, sa.Name, err)
}
// continue watching for good tokens
return false, nil
}
clientConfig = validConfig
return true, nil
default:
return false, fmt.Errorf("unexpected event type: %v", event.Type)
}
})
if err != nil {
return nil, fmt.Errorf("unable to get token for service account: %v", err)
}
return clientConfig, nil
}
func (b SAControllerClientBuilder) getAuthenticatedConfig(sa *v1.ServiceAccount, token string) (*restclient.Config, bool, error) {
username := apiserverserviceaccount.MakeUsername(sa.Namespace, sa.Name)
clientConfig := restclient.AnonymousClientConfig(b.ClientConfig)
clientConfig.BearerToken = token
restclient.AddUserAgent(clientConfig, username)
// Try token review first
tokenReview := &v1authenticationapi.TokenReview{Spec: v1authenticationapi.TokenReviewSpec{Token: token}}
if tokenResult, err := b.AuthenticationClient.TokenReviews().Create(context.TODO(), tokenReview, metav1.CreateOptions{}); err == nil {
if !tokenResult.Status.Authenticated {
klog.Warningf("Token for %s/%s did not authenticate correctly", sa.Namespace, sa.Name)
return nil, false, nil
}
if tokenResult.Status.User.Username != username {
klog.Warningf("Token for %s/%s authenticated as unexpected username: %s", sa.Namespace, sa.Name, tokenResult.Status.User.Username)
return nil, false, nil
}
klog.V(4).Infof("Verified credential for %s/%s", sa.Namespace, sa.Name)
return clientConfig, true, nil
}
// If we couldn't run the token review, the API might be disabled or we might not have permission.
// Try to make a request to /apis with the token. If we get a 401 we should consider the token invalid.
clientConfigCopy := *clientConfig
clientConfigCopy.NegotiatedSerializer = scheme.Codecs
client, err := restclient.UnversionedRESTClientFor(&clientConfigCopy)
if err != nil {
return nil, false, err
}
err = client.Get().AbsPath("/apis").Do(context.TODO()).Error()
if apierrors.IsUnauthorized(err) {
klog.Warningf("Token for %s/%s did not authenticate correctly: %v", sa.Namespace, sa.Name, err)
return nil, false, nil
}
return clientConfig, true, nil
}
// ConfigOrDie returns clientConfig for constructing clients.
// If it gets an error, it will log the error and kill the process it's running in.
func (b SAControllerClientBuilder) ConfigOrDie(name string) *restclient.Config {
clientConfig, err := b.Config(name)
if err != nil {
klog.Fatal(err)
}
return clientConfig
}
// Client returns clientset.Interface built from ClientBuilder
func (b SAControllerClientBuilder) Client(name string) (clientset.Interface, error) {
clientConfig, err := b.Config(name)
if err != nil {
return nil, err
}
return clientset.NewForConfig(clientConfig)
}
// ClientOrDie will return clientset.Interface built from ClientBuilder.
// If it gets an error getting the client, it will log the error and kill the process it's running in.
func (b SAControllerClientBuilder) ClientOrDie(name string) clientset.Interface {
client, err := b.Client(name)
if err != nil {
klog.Fatal(err)
}
return client
}

View File

@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
package controller
package clientbuilder
import (
"context"
@ -25,6 +25,8 @@ import (
"golang.org/x/oauth2"
v1authenticationapi "k8s.io/api/authentication/v1"
v1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/clock"
"k8s.io/apimachinery/pkg/util/wait"
@ -33,7 +35,6 @@ import (
v1core "k8s.io/client-go/kubernetes/typed/core/v1"
restclient "k8s.io/client-go/rest"
"k8s.io/client-go/transport"
"k8s.io/controller-manager/pkg/clientbuilder"
"k8s.io/klog/v2"
utilpointer "k8s.io/utils/pointer"
)
@ -73,7 +74,8 @@ type DynamicControllerClientBuilder struct {
clock clock.Clock
}
func NewDynamicClientBuilder(clientConfig *restclient.Config, coreClient v1core.CoreV1Interface, ns string) clientbuilder.ControllerClientBuilder {
// NewDynamicClientBuilder returns client builder which uses TokenRequest feature and refresh service account token periodically
func NewDynamicClientBuilder(clientConfig *restclient.Config, coreClient v1core.CoreV1Interface, ns string) ControllerClientBuilder {
builder := &DynamicControllerClientBuilder{
ClientConfig: clientConfig,
CoreClient: coreClient,
@ -87,7 +89,7 @@ func NewDynamicClientBuilder(clientConfig *restclient.Config, coreClient v1core.
}
// this function only for test purpose, don't call it
func NewTestDynamicClientBuilder(clientConfig *restclient.Config, coreClient v1core.CoreV1Interface, ns string, expirationSeconds int64, leewayPercent int) clientbuilder.ControllerClientBuilder {
func NewTestDynamicClientBuilder(clientConfig *restclient.Config, coreClient v1core.CoreV1Interface, ns string, expirationSeconds int64, leewayPercent int) ControllerClientBuilder {
builder := &DynamicControllerClientBuilder{
ClientConfig: clientConfig,
CoreClient: coreClient,
@ -217,3 +219,29 @@ func constructClient(saNamespace, saName string, config *restclient.Config) rest
restclient.AddUserAgent(&ret, username)
return ret
}
func getOrCreateServiceAccount(coreClient v1core.CoreV1Interface, namespace, name string) (*v1.ServiceAccount, error) {
sa, err := coreClient.ServiceAccounts(namespace).Get(context.TODO(), name, metav1.GetOptions{})
if err == nil {
return sa, nil
}
if !apierrors.IsNotFound(err) {
return nil, err
}
// Create the namespace if we can't verify it exists.
// Tolerate errors, since we don't know whether this component has namespace creation permissions.
if _, err := coreClient.Namespaces().Get(context.TODO(), namespace, metav1.GetOptions{}); apierrors.IsNotFound(err) {
if _, err = coreClient.Namespaces().Create(context.TODO(), &v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: namespace}}, metav1.CreateOptions{}); err != nil && !apierrors.IsAlreadyExists(err) {
klog.Warningf("create non-exist namespace %s failed:%v", namespace, err)
}
}
// Create the service account
sa, err = coreClient.ServiceAccounts(namespace).Create(context.TODO(), &v1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: name}}, metav1.CreateOptions{})
if apierrors.IsAlreadyExists(err) {
// If we're racing to init and someone else already created it, re-fetch
return coreClient.ServiceAccounts(namespace).Get(context.TODO(), name, metav1.GetOptions{})
}
return sa, err
}

View File

@ -29,7 +29,6 @@ go_test(
"//pkg/apis/extensions:go_default_library",
"//pkg/apis/rbac/v1:go_default_library",
"//pkg/auth/authorizer/abac:go_default_library",
"//pkg/controller:go_default_library",
"//pkg/controller/serviceaccount:go_default_library",
"//pkg/controlplane:go_default_library",
"//pkg/features:go_default_library",
@ -86,6 +85,7 @@ go_test(
"//staging/src/k8s.io/client-go/util/keyutil:go_default_library",
"//staging/src/k8s.io/cluster-bootstrap/token/api:go_default_library",
"//staging/src/k8s.io/component-base/featuregate/testing:go_default_library",
"//staging/src/k8s.io/controller-manager/pkg/clientbuilder:go_default_library",
"//test/e2e/lifecycle/bootstrap:go_default_library",
"//test/integration:go_default_library",
"//test/integration/framework:go_default_library",

View File

@ -28,8 +28,8 @@ import (
"k8s.io/apiserver/pkg/authorization/authorizerfactory"
clientset "k8s.io/client-go/kubernetes"
restclient "k8s.io/client-go/rest"
"k8s.io/controller-manager/pkg/clientbuilder"
"k8s.io/kubernetes/cmd/kube-apiserver/app/options"
"k8s.io/kubernetes/pkg/controller"
"k8s.io/kubernetes/pkg/controlplane"
kubeoptions "k8s.io/kubernetes/pkg/kubeapiserver/options"
"k8s.io/kubernetes/test/integration/framework"
@ -84,7 +84,7 @@ func TestDynamicClientBuilder(t *testing.T) {
exp := int64(600)
leeway := 99
ns := "default"
clientBuilder := controller.NewTestDynamicClientBuilder(
clientBuilder := clientbuilder.NewTestDynamicClientBuilder(
restclient.AnonymousClientConfig(baseConfig),
baseClient.CoreV1(),
ns, exp, leeway)