diff --git a/staging/src/k8s.io/dynamic-resource-allocation/controller/controller.go b/staging/src/k8s.io/dynamic-resource-allocation/controller/controller.go new file mode 100644 index 00000000000..d0d44858e8e --- /dev/null +++ b/staging/src/k8s.io/dynamic-resource-allocation/controller/controller.go @@ -0,0 +1,782 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + "time" + + "github.com/go-logr/logr" + "github.com/google/go-cmp/cmp" + + v1 "k8s.io/api/core/v1" + resourcev1alpha1 "k8s.io/api/resource/v1alpha1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/scheme" + corev1types "k8s.io/client-go/kubernetes/typed/core/v1" + resourcev1alpha1listers "k8s.io/client-go/listers/resource/v1alpha1" + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/tools/record" + "k8s.io/client-go/util/workqueue" + "k8s.io/dynamic-resource-allocation/resourceclaim" + "k8s.io/klog/v2" +) + +// Controller watches ResourceClaims and triggers allocation and deallocation +// as needed. +type Controller interface { + // Run starts the controller. + Run(workers int) +} + +// Driver provides the actual allocation and deallocation operations. +type Driver interface { + // GetClassParameters gets called to retrieve the parameter object + // referenced by a class. The content should be validated now if + // possible. class.Parameters may be nil. + // + // The caller will wrap the error to include the parameter reference. + GetClassParameters(ctx context.Context, class *resourcev1alpha1.ResourceClass) (interface{}, error) + + // GetClaimParameters gets called to retrieve the parameter object + // referenced by a claim. The content should be validated now if + // possible. claim.Spec.Parameters may be nil. + // + // The caller will wrap the error to include the parameter reference. + GetClaimParameters(ctx context.Context, claim *resourcev1alpha1.ResourceClaim, class *resourcev1alpha1.ResourceClass, classParameters interface{}) (interface{}, error) + + // Allocate gets called when a ResourceClaim is ready to be allocated. + // The selectedNode is empty for ResourceClaims with immediate + // allocation, in which case the resource driver decides itself where + // to allocate. If there is already an on-going allocation, the driver + // may finish it and ignore the new parameters or abort the on-going + // allocation and try again with the new parameters. + // + // Parameters have been retrieved earlier. + // + // If selectedNode is set, the driver must attempt to allocate for that + // node. If that is not possible, it must return an error. The + // controller will call UnsuitableNodes and pass the new information to + // the scheduler, which then will lead to selecting a diffent node + // if the current one is not suitable. + // + // The objects are read-only and must not be modified. This call + // must be idempotent. + Allocate(ctx context.Context, claim *resourcev1alpha1.ResourceClaim, claimParameters interface{}, class *resourcev1alpha1.ResourceClass, classParameters interface{}, selectedNode string) (*resourcev1alpha1.AllocationResult, error) + + // Deallocate gets called when a ResourceClaim is ready to be + // freed. + // + // The claim is read-only and must not be modified. This call must be + // idempotent. In particular it must not return an error when the claim + // is currently not allocated. + // + // Deallocate may get called when a previous allocation got + // interrupted. Deallocate must then stop any on-going allocation + // activity and free resources before returning without an error. + Deallocate(ctx context.Context, claim *resourcev1alpha1.ResourceClaim) error + + // UnsuitableNodes checks all pending claims with delayed allocation + // for a pod. All claims are ready for allocation by the driver + // and parameters have been retrieved. + // + // The driver may consider each claim in isolation, but it's better + // to mark nodes as unsuitable for all claims if it not all claims + // can be allocated for it (for example, two GPUs requested but + // the node only has one). + // + // The result of the check is in ClaimAllocation.UnsuitableNodes. + // An error indicates that the entire check must be repeated. + UnsuitableNodes(ctx context.Context, pod *v1.Pod, claims []*ClaimAllocation, potentialNodes []string) error +} + +// ClaimAllocation represents information about one particular +// pod.Spec.ResourceClaim entry. +type ClaimAllocation struct { + PodClaimName string + Claim *resourcev1alpha1.ResourceClaim + Class *resourcev1alpha1.ResourceClass + ClaimParameters interface{} + ClassParameters interface{} + + // UnsuitableNodes needs to be filled in by the driver when + // Driver.UnsuitableNodes gets called. + UnsuitableNodes []string +} + +type controller struct { + ctx context.Context + logger klog.Logger + name string + finalizer string + driver Driver + kubeClient kubernetes.Interface + queue workqueue.RateLimitingInterface + eventRecorder record.EventRecorder + rcLister resourcev1alpha1listers.ResourceClassLister + rcSynced cache.InformerSynced + claimLister resourcev1alpha1listers.ResourceClaimLister + podSchedulingLister resourcev1alpha1listers.PodSchedulingLister + claimSynced cache.InformerSynced + podSchedulingSynced cache.InformerSynced +} + +// TODO: make it configurable +var recheckDelay = 30 * time.Second + +// New creates a new controller. +func New( + ctx context.Context, + name string, + driver Driver, + kubeClient kubernetes.Interface, + informerFactory informers.SharedInformerFactory) Controller { + logger := klog.LoggerWithName(klog.FromContext(ctx), "resource controller") + rcInformer := informerFactory.Resource().V1alpha1().ResourceClasses() + claimInformer := informerFactory.Resource().V1alpha1().ResourceClaims() + podSchedulingInformer := informerFactory.Resource().V1alpha1().PodSchedulings() + + eventBroadcaster := record.NewBroadcaster() + // TODO: use contextual logging in eventBroadcaster once it + // supports it. There is a StartStructuredLogging API, but it + // uses the global klog, which is worse than redirecting an unstructured + // string into our logger, in particular during testing. + eventBroadcaster.StartLogging(func(format string, args ...interface{}) { + helper, logger := logger.WithCallStackHelper() + helper() + logger.V(2).Info(fmt.Sprintf(format, args...)) + }) + eventBroadcaster.StartRecordingToSink(&corev1types.EventSinkImpl{Interface: kubeClient.CoreV1().Events(v1.NamespaceAll)}) + eventRecorder := eventBroadcaster.NewRecorder(scheme.Scheme, + v1.EventSource{Component: fmt.Sprintf("resource driver %s", name)}) + + // The work queue contains either keys for claims or PodScheduling objects. + queue := workqueue.NewNamedRateLimitingQueue( + workqueue.DefaultControllerRateLimiter(), fmt.Sprintf("%s-queue", name)) + + ctrl := &controller{ + ctx: ctx, + logger: logger, + name: name, + finalizer: name + "/deletion-protection", + driver: driver, + kubeClient: kubeClient, + rcLister: rcInformer.Lister(), + rcSynced: rcInformer.Informer().HasSynced, + claimLister: claimInformer.Lister(), + claimSynced: claimInformer.Informer().HasSynced, + podSchedulingLister: podSchedulingInformer.Lister(), + podSchedulingSynced: podSchedulingInformer.Informer().HasSynced, + queue: queue, + eventRecorder: eventRecorder, + } + + loggerV6 := logger.V(6) + if loggerV6.Enabled() { + resourceClaimLogger := klog.LoggerWithValues(loggerV6, "type", "ResourceClaim") + _, _ = claimInformer.Informer().AddEventHandler(resourceEventHandlerFuncs(&resourceClaimLogger, ctrl)) + podSchedulingLogger := klog.LoggerWithValues(loggerV6, "type", "PodScheduling") + _, _ = podSchedulingInformer.Informer().AddEventHandler(resourceEventHandlerFuncs(&podSchedulingLogger, ctrl)) + } else { + _, _ = claimInformer.Informer().AddEventHandler(resourceEventHandlerFuncs(nil, ctrl)) + _, _ = podSchedulingInformer.Informer().AddEventHandler(resourceEventHandlerFuncs(nil, ctrl)) + } + + return ctrl +} + +func resourceEventHandlerFuncs(logger *klog.Logger, ctrl *controller) cache.ResourceEventHandlerFuncs { + return cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { + ctrl.add(logger, obj) + }, + UpdateFunc: func(oldObj, newObj interface{}) { + ctrl.update(logger, oldObj, newObj) + }, + DeleteFunc: ctrl.delete, + } +} + +const ( + claimKeyPrefix = "claim:" + podSchedulingKeyPrefix = "podscheduling:" +) + +func (ctrl *controller) add(logger *klog.Logger, obj interface{}) { + if logger != nil { + logger.Info("new object", "content", prettyPrint(obj)) + } + ctrl.addNewOrUpdated("Adding new work item", obj) +} + +func (ctrl *controller) update(logger *klog.Logger, oldObj, newObj interface{}) { + if logger != nil { + diff := cmp.Diff(oldObj, newObj) + logger.Info("updated object", "content", prettyPrint(newObj), "diff", diff) + } + ctrl.addNewOrUpdated("Adding updated work item", newObj) +} + +func (ctrl *controller) addNewOrUpdated(msg string, obj interface{}) { + objKey, err := getKey(obj) + if err != nil { + ctrl.logger.Error(err, "Failed to get key", "obj", obj) + return + } + ctrl.logger.V(5).Info(msg, "key", objKey) + ctrl.queue.Add(objKey) +} + +func (ctrl *controller) delete(obj interface{}) { + objKey, err := getKey(obj) + if err != nil { + return + } + ctrl.logger.V(5).Info("Removing deleted work item", "key", objKey) + ctrl.queue.Forget(objKey) +} + +func getKey(obj interface{}) (string, error) { + objKey, err := cache.DeletionHandlingMetaNamespaceKeyFunc(obj) + if err != nil { + return "", err + } + prefix := "" + switch obj.(type) { + case *resourcev1alpha1.ResourceClaim: + prefix = claimKeyPrefix + case *resourcev1alpha1.PodScheduling: + prefix = podSchedulingKeyPrefix + default: + return "", fmt.Errorf("unexpected object: %T", obj) + } + + return prefix + objKey, nil +} + +// Run starts the controller. +func (ctrl *controller) Run(workers int) { + defer ctrl.queue.ShutDown() + + ctrl.logger.Info("Starting", "driver", ctrl.name) + defer ctrl.logger.Info("Shutting down", "driver", ctrl.name) + + stopCh := ctrl.ctx.Done() + + if !cache.WaitForCacheSync(stopCh, ctrl.rcSynced, ctrl.claimSynced, ctrl.podSchedulingSynced) { + ctrl.logger.Error(nil, "Cannot sync caches") + return + } + + for i := 0; i < workers; i++ { + go wait.Until(ctrl.sync, 0, stopCh) + } + + <-stopCh +} + +// errRequeue is a special error instance that functions can return +// to request silent requeueing (not logged as error, no event). +// Uses exponential backoff. +var errRequeue = errors.New("requeue") + +// errPeriodic is a special error instance that functions can return +// to request silent instance that functions can return +// to request silent retrying at a fixed rate. +var errPeriodic = errors.New("periodic") + +// sync is the main worker. +func (ctrl *controller) sync() { + key, quit := ctrl.queue.Get() + if quit { + return + } + defer ctrl.queue.Done(key) + + logger := klog.LoggerWithValues(ctrl.logger, "key", key) + ctx := klog.NewContext(ctrl.ctx, logger) + logger.V(4).Info("processing") + obj, err := ctrl.syncKey(ctx, key.(string)) + switch err { + case nil: + logger.V(5).Info("completed") + ctrl.queue.Forget(key) + case errRequeue: + logger.V(5).Info("requeue") + ctrl.queue.AddRateLimited(key) + case errPeriodic: + logger.V(5).Info("recheck periodically") + ctrl.queue.AddAfter(key, recheckDelay) + default: + logger.Error(err, "processing failed") + if obj != nil { + // TODO: We don't know here *what* failed. Determine based on error? + ctrl.eventRecorder.Event(obj, v1.EventTypeWarning, "Failed", err.Error()) + } + ctrl.queue.AddRateLimited(key) + } +} + +// syncKey looks up a ResourceClaim by its key and processes it. +func (ctrl *controller) syncKey(ctx context.Context, key string) (obj runtime.Object, finalErr error) { + sep := strings.Index(key, ":") + if sep < 0 { + return nil, fmt.Errorf("unexpected key: %s", key) + } + prefix, object := key[0:sep+1], key[sep+1:] + namespace, name, err := cache.SplitMetaNamespaceKey(object) + if err != nil { + return nil, err + } + + switch prefix { + case claimKeyPrefix: + claim, err := ctrl.claimLister.ResourceClaims(namespace).Get(name) + if err != nil { + if k8serrors.IsNotFound(err) { + klog.FromContext(ctx).V(5).Info("ResourceClaim was deleted, no need to process it") + return nil, nil + } + return nil, err + } + obj, finalErr = claim, ctrl.syncClaim(ctx, claim) + case podSchedulingKeyPrefix: + podScheduling, err := ctrl.podSchedulingLister.PodSchedulings(namespace).Get(name) + if err != nil { + if k8serrors.IsNotFound(err) { + klog.FromContext(ctx).V(5).Info("PodScheduling was deleted, no need to process it") + return nil, nil + } + return nil, err + } + obj, finalErr = podScheduling, ctrl.syncPodScheduling(ctx, podScheduling) + } + return +} + +// syncClaim determines which next action may be needed for a ResourceClaim +// and does it. +func (ctrl *controller) syncClaim(ctx context.Context, claim *resourcev1alpha1.ResourceClaim) error { + var err error + logger := klog.FromContext(ctx) + + if len(claim.Status.ReservedFor) > 0 { + // In use. Nothing that we can do for it now. + if loggerV6 := logger.V(6); loggerV6.Enabled() { + loggerV6.Info("ResourceClaim in use", "reservedFor", claim.Status.ReservedFor) + } else { + logger.V(5).Info("ResourceClaim in use") + } + return nil + } + + if claim.DeletionTimestamp != nil || + claim.Status.DeallocationRequested { + // Ready for deallocation. We might have our finalizer set. The + // finalizer is specific to the driver, therefore we know that + // this claim is "ours" when the finalizer is set. + hasFinalizer := ctrl.hasFinalizer(claim) + logger.V(5).Info("ResourceClaim ready for deallocation", "deallocationRequested", claim.Status.DeallocationRequested, "deletionTimestamp", claim.DeletionTimestamp, "allocated", claim.Status.Allocation != nil, "hasFinalizer", hasFinalizer) + if hasFinalizer { + claim = claim.DeepCopy() + if claim.Status.Allocation != nil { + // Allocation was completed. Deallocate before proceeding. + if err := ctrl.driver.Deallocate(ctx, claim); err != nil { + return fmt.Errorf("deallocate: %v", err) + } + claim.Status.Allocation = nil + claim.Status.DriverName = "" + claim.Status.DeallocationRequested = false + claim, err = ctrl.kubeClient.ResourceV1alpha1().ResourceClaims(claim.Namespace).UpdateStatus(ctx, claim, metav1.UpdateOptions{}) + if err != nil { + return fmt.Errorf("remove allocation: %v", err) + } + } else { + // Ensure that there is no on-going allocation. + if err := ctrl.driver.Deallocate(ctx, claim); err != nil { + return fmt.Errorf("stop allocation: %v", err) + } + } + + if claim.Status.DeallocationRequested { + // Still need to remove it. + claim.Status.DeallocationRequested = false + claim, err = ctrl.kubeClient.ResourceV1alpha1().ResourceClaims(claim.Namespace).UpdateStatus(ctx, claim, metav1.UpdateOptions{}) + if err != nil { + return fmt.Errorf("remove deallocation: %v", err) + } + } + + claim.Finalizers = ctrl.removeFinalizer(claim.Finalizers) + if _, err := ctrl.kubeClient.ResourceV1alpha1().ResourceClaims(claim.Namespace).Update(ctx, claim, metav1.UpdateOptions{}); err != nil { + return fmt.Errorf("remove finalizer: %v", err) + } + } + + // Nothing further to do. The apiserver should remove it shortly. + return nil + + } + + if claim.Status.Allocation != nil { + logger.V(5).Info("ResourceClaim is allocated") + return nil + } + if claim.Spec.AllocationMode != resourcev1alpha1.AllocationModeImmediate { + logger.V(5).Info("ResourceClaim waiting for first consumer") + return nil + } + + // We need the ResourceClass to determine whether we should allocate it. + class, err := ctrl.rcLister.Get(claim.Spec.ResourceClassName) + if err != nil { + return err + } + if class.DriverName != ctrl.name { + // Not ours *at the moment*. This can change, so requeue and + // check again. We could trigger a faster check when the + // ResourceClass changes, but that shouldn't occur much in + // practice and thus isn't worth the effort. + // + // We use exponential backoff because it is unlikely that + // the ResourceClass changes much. + logger.V(5).Info("ResourceClaim is handled by other driver", "driver", class.DriverName) + return errRequeue + } + + // Check parameters. + claimParameters, classParameters, err := ctrl.getParameters(ctx, claim, class) + if err != nil { + return err + } + + return ctrl.allocateClaim(ctx, claim, claimParameters, class, classParameters, "", nil) +} + +func (ctrl *controller) getParameters(ctx context.Context, claim *resourcev1alpha1.ResourceClaim, class *resourcev1alpha1.ResourceClass) (claimParameters, classParameters interface{}, err error) { + classParameters, err = ctrl.driver.GetClassParameters(ctx, class) + if err != nil { + err = fmt.Errorf("class parameters %s: %v", class.ParametersRef, err) + return + } + claimParameters, err = ctrl.driver.GetClaimParameters(ctx, claim, class, classParameters) + if err != nil { + err = fmt.Errorf("claim parameters %s: %v", claim.Spec.ParametersRef, err) + return + } + return +} + +func (ctrl *controller) allocateClaim(ctx context.Context, + claim *resourcev1alpha1.ResourceClaim, claimParameters interface{}, + class *resourcev1alpha1.ResourceClass, classParameters interface{}, + selectedNode string, + selectedUser *resourcev1alpha1.ResourceClaimConsumerReference) error { + logger := klog.FromContext(ctx) + + if claim.Status.Allocation != nil { + // This can happen when two PodScheduling objects trigger + // allocation attempts (first one wins) or when we see the + // update of the PodScheduling object. + logger.V(5).Info("Claim already allocated, nothing to do") + return nil + } + + claim = claim.DeepCopy() + if !ctrl.hasFinalizer(claim) { + // Set finalizer before doing anything. We continue with the updated claim. + logger.V(5).Info("Adding finalizer") + claim.Finalizers = append(claim.Finalizers, ctrl.finalizer) + var err error + claim, err = ctrl.kubeClient.ResourceV1alpha1().ResourceClaims(claim.Namespace).Update(ctx, claim, metav1.UpdateOptions{}) + if err != nil { + return fmt.Errorf("add finalizer: %v", err) + } + } + + logger.V(5).Info("Allocating") + allocation, err := ctrl.driver.Allocate(ctx, claim, claimParameters, class, classParameters, selectedNode) + if err != nil { + return fmt.Errorf("allocate: %v", err) + } + claim.Status.Allocation = allocation + claim.Status.DriverName = ctrl.name + if selectedUser != nil { + claim.Status.ReservedFor = append(claim.Status.ReservedFor, *selectedUser) + } + logger.V(6).Info("Updating claim after allocation", "claim", claim) + if _, err := ctrl.kubeClient.ResourceV1alpha1().ResourceClaims(claim.Namespace).UpdateStatus(ctx, claim, metav1.UpdateOptions{}); err != nil { + return fmt.Errorf("add allocation: %v", err) + } + return nil +} + +func (ctrl *controller) checkPodClaim(ctx context.Context, pod *v1.Pod, podClaim v1.PodResourceClaim) (*ClaimAllocation, error) { + claimName := resourceclaim.Name(pod, &podClaim) + claim, err := ctrl.claimLister.ResourceClaims(pod.Namespace).Get(claimName) + if err != nil { + return nil, err + } + if podClaim.Source.ResourceClaimTemplateName != nil { + if err := resourceclaim.IsForPod(pod, claim); err != nil { + return nil, err + } + } + if claim.Spec.AllocationMode != resourcev1alpha1.AllocationModeWaitForFirstConsumer { + // Nothing to do for it as part of pod scheduling. + return nil, nil + } + class, err := ctrl.rcLister.Get(claim.Spec.ResourceClassName) + if err != nil { + return nil, err + } + if class.DriverName != ctrl.name { + return nil, nil + } + // Check parameters. + claimParameters, classParameters, err := ctrl.getParameters(ctx, claim, class) + if err != nil { + return nil, err + } + return &ClaimAllocation{ + PodClaimName: podClaim.Name, + Claim: claim, + Class: class, + ClaimParameters: claimParameters, + ClassParameters: classParameters, + }, nil +} + +// syncClaim determines which next action may be needed for a PodScheduling object +// and does it. +func (ctrl *controller) syncPodScheduling(ctx context.Context, podScheduling *resourcev1alpha1.PodScheduling) error { + logger := klog.FromContext(ctx) + + // Ignore deleted objects. + if podScheduling.DeletionTimestamp != nil { + logger.V(5).Info("PodScheduling marked for deletion") + return nil + } + + if podScheduling.Spec.SelectedNode == "" && + len(podScheduling.Spec.PotentialNodes) == 0 { + // Nothing to do? Shouldn't occur. + logger.V(5).Info("Waiting for scheduler to set fields") + return nil + } + + // Check pod. + // TODO (?): use an informer - only useful when many (most?) pods have claims + // TODO (?): let the scheduler copy all claim names + UIDs into PodScheduling - then we don't need the pod + pod, err := ctrl.kubeClient.CoreV1().Pods(podScheduling.Namespace).Get(ctx, podScheduling.Name, metav1.GetOptions{}) + if err != nil { + return err + } + if pod.DeletionTimestamp != nil { + logger.V(5).Info("Pod marked for deletion") + return nil + } + + // Still the owner? + if !metav1.IsControlledBy(podScheduling, pod) { + // Must be obsolete object, do nothing for it. + logger.V(5).Info("Pod not owner, PodScheduling is obsolete") + return nil + } + + // Find all pending claims that are owned by us. We bail out if any of the pre-requisites + // for pod scheduling (claims exist, classes exist, parameters exist) are not met. + // The scheduler will do the same, except for checking parameters, so usually + // everything should be ready once the PodScheduling object exists. + var claims claimAllocations + for _, podClaim := range pod.Spec.ResourceClaims { + delayed, err := ctrl.checkPodClaim(ctx, pod, podClaim) + if err != nil { + return fmt.Errorf("pod claim %s: %v", podClaim.Name, err) + } + if delayed == nil { + // Nothing to do for it. This can change, so keep checking. + continue + } + claims = append(claims, delayed) + } + if len(claims) == 0 { + logger.V(5).Info("Found no pending pod claims") + return errPeriodic + } + + // Check current resource availability *before* triggering the + // allocations. If we find that any of the claims cannot be allocated + // for the selected node, we don't need to try for the others either + // and shouldn't, because those allocations might have to be undone to + // pick a better node. If we don't need to allocate now, then we'll + // simply report back the gather information. + if len(podScheduling.Spec.PotentialNodes) > 0 { + if err := ctrl.driver.UnsuitableNodes(ctx, pod, claims, podScheduling.Spec.PotentialNodes); err != nil { + return fmt.Errorf("checking potential nodes: %v", err) + } + } + selectedNode := podScheduling.Spec.SelectedNode + logger.V(5).Info("pending pod claims", "claims", claims, "selectedNode", selectedNode) + if selectedNode != "" { + unsuitable := false + for _, delayed := range claims { + if hasString(delayed.UnsuitableNodes, selectedNode) { + unsuitable = true + break + } + } + + if unsuitable { + logger.V(2).Info("skipping allocation for unsuitable selected node", "node", selectedNode) + } else { + logger.V(2).Info("allocation for selected node", "node", selectedNode) + selectedUser := &resourcev1alpha1.ResourceClaimConsumerReference{ + Resource: "pods", + Name: pod.Name, + UID: pod.UID, + } + for _, delayed := range claims { + if err := ctrl.allocateClaim(ctx, delayed.Claim, delayed.ClaimParameters, delayed.Class, delayed.ClassParameters, selectedNode, selectedUser); err != nil { + return fmt.Errorf("allocation of pod claim %s failed: %v", delayed.PodClaimName, err) + } + } + } + } + + // Now update unsuitable nodes. This is useful information for the scheduler even if + // we managed to allocate because we might have to undo that. + // TODO: replace with patching the array. We can do that without race conditions + // because each driver is responsible for its own entries. + modified := false + podScheduling = podScheduling.DeepCopy() + for _, delayed := range claims { + i := findClaim(podScheduling.Status.ResourceClaims, delayed.PodClaimName) + if i < 0 { + // Add new entry. + podScheduling.Status.ResourceClaims = append(podScheduling.Status.ResourceClaims, + resourcev1alpha1.ResourceClaimSchedulingStatus{ + Name: delayed.PodClaimName, + UnsuitableNodes: delayed.UnsuitableNodes, + }) + modified = true + } else if stringsDiffer(podScheduling.Status.ResourceClaims[i].UnsuitableNodes, delayed.UnsuitableNodes) { + // Update existing entry. + podScheduling.Status.ResourceClaims[i].UnsuitableNodes = delayed.UnsuitableNodes + modified = true + } + } + if modified { + logger.V(6).Info("Updating pod scheduling with modified unsuitable nodes", "podScheduling", podScheduling) + if _, err := ctrl.kubeClient.ResourceV1alpha1().PodSchedulings(podScheduling.Namespace).UpdateStatus(ctx, podScheduling, metav1.UpdateOptions{}); err != nil { + return fmt.Errorf("update unsuitable node status: %v", err) + } + } + + // We must keep the object in our queue and keep updating the + // UnsuitableNodes fields. + return errPeriodic +} + +type claimAllocations []*ClaimAllocation + +// MarshalLog replaces the pointers with the actual structs because +// we care about the content, not the pointer values. +func (claims claimAllocations) MarshalLog() interface{} { + content := make([]ClaimAllocation, 0, len(claims)) + for _, claim := range claims { + content = append(content, *claim) + } + return content +} + +var _ logr.Marshaler = claimAllocations{} + +// findClaim returns the index of the specified pod claim, -1 if not found. +func findClaim(claims []resourcev1alpha1.ResourceClaimSchedulingStatus, podClaimName string) int { + for i := range claims { + if claims[i].Name == podClaimName { + return i + } + } + return -1 +} + +// hasString checks for a string in a slice. +func hasString(strings []string, str string) bool { + for _, s := range strings { + if s == str { + return true + } + } + return false +} + +// stringsDiffer does a strict comparison of two string arrays, order of entries matters. +func stringsDiffer(a, b []string) bool { + if len(a) != len(b) { + return true + } + for i := range a { + if a[i] != b[i] { + return true + } + } + return false +} + +// hasFinalizer checks if the claim has the finalizer of the driver. +func (ctrl *controller) hasFinalizer(claim *resourcev1alpha1.ResourceClaim) bool { + for _, finalizer := range claim.Finalizers { + if finalizer == ctrl.finalizer { + return true + } + } + return false +} + +// removeFinalizer creates a new slice without the finalizer of the driver. +func (ctrl *controller) removeFinalizer(in []string) []string { + out := make([]string, 0, len(in)) + for _, finalizer := range in { + if finalizer != ctrl.finalizer { + out = append(out, finalizer) + } + } + if len(out) == 0 { + return nil + } + return out +} + +// prettyPrint formats arbitrary objects as JSON or, if that fails, with Sprintf. +func prettyPrint(obj interface{}) string { + buffer, err := json.Marshal(obj) + if err != nil { + return fmt.Sprintf("%s", obj) + } + return string(buffer) +} diff --git a/staging/src/k8s.io/dynamic-resource-allocation/controller/controller_test.go b/staging/src/k8s.io/dynamic-resource-allocation/controller/controller_test.go new file mode 100644 index 00000000000..0aeb2610601 --- /dev/null +++ b/staging/src/k8s.io/dynamic-resource-allocation/controller/controller_test.go @@ -0,0 +1,637 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "errors" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + corev1 "k8s.io/api/core/v1" + resourcev1alpha1 "k8s.io/api/resource/v1alpha1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/fake" + "k8s.io/klog/v2/ktesting" + _ "k8s.io/klog/v2/ktesting/init" +) + +func TestController(t *testing.T) { + claimKey := "claim:default/claim" + claimName := "claim" + claimNamespace := "default" + driverName := "mock-driver" + className := "mock-class" + otherDriverName := "other-driver" + otherClassName := "other-class" + ourFinalizer := driverName + "/deletion-protection" + otherFinalizer := otherDriverName + "/deletion-protection" + classes := []*resourcev1alpha1.ResourceClass{ + createClass(className, driverName), + createClass(otherClassName, otherDriverName), + } + claim := createClaim(claimName, claimNamespace, className) + otherClaim := createClaim(claimName, claimNamespace, otherClassName) + delayedClaim := claim.DeepCopy() + delayedClaim.Spec.AllocationMode = resourcev1alpha1.AllocationModeWaitForFirstConsumer + podName := "pod" + podKey := "podscheduling:default/pod" + pod := createPod(podName, claimNamespace, nil) + podClaimName := "my-pod-claim" + podScheduling := createPodScheduling(pod) + podWithClaim := createPod(podName, claimNamespace, map[string]string{podClaimName: claimName}) + nodeName := "worker" + otherNodeName := "worker-2" + unsuitableNodes := []string{otherNodeName} + potentialNodes := []string{nodeName, otherNodeName} + withDeletionTimestamp := func(claim *resourcev1alpha1.ResourceClaim) *resourcev1alpha1.ResourceClaim { + var deleted metav1.Time + claim = claim.DeepCopy() + claim.DeletionTimestamp = &deleted + return claim + } + withReservedFor := func(claim *resourcev1alpha1.ResourceClaim, pod *corev1.Pod) *resourcev1alpha1.ResourceClaim { + claim = claim.DeepCopy() + claim.Status.ReservedFor = append(claim.Status.ReservedFor, resourcev1alpha1.ResourceClaimConsumerReference{ + Resource: "pods", + Name: pod.Name, + UID: pod.UID, + }) + return claim + } + withFinalizer := func(claim *resourcev1alpha1.ResourceClaim, finalizer string) *resourcev1alpha1.ResourceClaim { + claim = claim.DeepCopy() + claim.Finalizers = append(claim.Finalizers, finalizer) + return claim + } + allocation := resourcev1alpha1.AllocationResult{} + withAllocate := func(claim *resourcev1alpha1.ResourceClaim) *resourcev1alpha1.ResourceClaim { + // Any allocated claim must have our finalizer. + claim = withFinalizer(claim, ourFinalizer) + claim.Status.Allocation = &allocation + claim.Status.DriverName = driverName + return claim + } + withDeallocate := func(claim *resourcev1alpha1.ResourceClaim) *resourcev1alpha1.ResourceClaim { + claim.Status.DeallocationRequested = true + return claim + } + withSelectedNode := func(podScheduling *resourcev1alpha1.PodScheduling) *resourcev1alpha1.PodScheduling { + podScheduling = podScheduling.DeepCopy() + podScheduling.Spec.SelectedNode = nodeName + return podScheduling + } + withUnsuitableNodes := func(podScheduling *resourcev1alpha1.PodScheduling) *resourcev1alpha1.PodScheduling { + podScheduling = podScheduling.DeepCopy() + podScheduling.Status.ResourceClaims = append(podScheduling.Status.ResourceClaims, + resourcev1alpha1.ResourceClaimSchedulingStatus{Name: podClaimName, UnsuitableNodes: unsuitableNodes}, + ) + return podScheduling + } + withPotentialNodes := func(podScheduling *resourcev1alpha1.PodScheduling) *resourcev1alpha1.PodScheduling { + podScheduling = podScheduling.DeepCopy() + podScheduling.Spec.PotentialNodes = potentialNodes + return podScheduling + } + + var m mockDriver + + for name, test := range map[string]struct { + key string + driver mockDriver + classes []*resourcev1alpha1.ResourceClass + pod *corev1.Pod + podScheduling, expectedPodScheduling *resourcev1alpha1.PodScheduling + claim, expectedClaim *resourcev1alpha1.ResourceClaim + expectedError string + }{ + "invalid-key": { + key: "claim:x/y/z", + expectedError: `unexpected key format: "x/y/z"`, + }, + "not-found": { + key: "claim:default/claim", + }, + "wrong-driver": { + key: claimKey, + classes: classes, + claim: otherClaim, + expectedClaim: otherClaim, + expectedError: errRequeue.Error(), // class might change + }, + // Immediate allocation: + // deletion time stamp set, our finalizer set, not allocated -> remove finalizer + "immediate-deleted-finalizer-removal": { + key: claimKey, + classes: classes, + claim: withFinalizer(withDeletionTimestamp(claim), ourFinalizer), + driver: m.expectDeallocate(map[string]error{claimName: nil}), + expectedClaim: withDeletionTimestamp(claim), + }, + // deletion time stamp set, our finalizer set, not allocated, stopping fails -> requeue + "immediate-deleted-finalizer-stop-failure": { + key: claimKey, + classes: classes, + claim: withFinalizer(withDeletionTimestamp(claim), ourFinalizer), + driver: m.expectDeallocate(map[string]error{claimName: errors.New("fake error")}), + expectedClaim: withFinalizer(withDeletionTimestamp(claim), ourFinalizer), + expectedError: "stop allocation: fake error", + }, + // deletion time stamp set, other finalizer set, not allocated -> do nothing + "immediate-deleted-finalizer-no-removal": { + key: claimKey, + classes: classes, + claim: withFinalizer(withDeletionTimestamp(claim), otherFinalizer), + expectedClaim: withFinalizer(withDeletionTimestamp(claim), otherFinalizer), + }, + // deletion time stamp set, finalizer set, allocated -> deallocate + "immediate-deleted-allocated": { + key: claimKey, + classes: classes, + claim: withAllocate(withDeletionTimestamp(claim)), + driver: m.expectDeallocate(map[string]error{claimName: nil}), + expectedClaim: withDeletionTimestamp(claim), + }, + // deletion time stamp set, finalizer set, allocated, deallocation fails -> requeue + "immediate-deleted-deallocate-failure": { + key: claimKey, + classes: classes, + claim: withAllocate(withDeletionTimestamp(claim)), + driver: m.expectDeallocate(map[string]error{claimName: errors.New("fake error")}), + expectedClaim: withAllocate(withDeletionTimestamp(claim)), + expectedError: "deallocate: fake error", + }, + // deletion time stamp set, finalizer not set -> do nothing + "immediate-deleted-no-finalizer": { + key: claimKey, + classes: classes, + claim: withDeletionTimestamp(claim), + expectedClaim: withDeletionTimestamp(claim), + }, + // not deleted, not allocated, no finalizer -> add finalizer, allocate + "immediate-do-allocation": { + key: claimKey, + classes: classes, + claim: claim, + driver: m.expectClassParameters(map[string]interface{}{className: 1}). + expectClaimParameters(map[string]interface{}{claimName: 2}). + expectAllocate(map[string]allocate{claimName: {allocResult: &allocation, allocErr: nil}}), + expectedClaim: withAllocate(claim), + }, + // not deleted, not allocated, finalizer -> allocate + "immediate-continue-allocation": { + key: claimKey, + classes: classes, + claim: withFinalizer(claim, ourFinalizer), + driver: m.expectClassParameters(map[string]interface{}{className: 1}). + expectClaimParameters(map[string]interface{}{claimName: 2}). + expectAllocate(map[string]allocate{claimName: {allocResult: &allocation, allocErr: nil}}), + expectedClaim: withAllocate(claim), + }, + // not deleted, not allocated, finalizer, fail allocation -> requeue + "immediate-fail-allocation": { + key: claimKey, + classes: classes, + claim: withFinalizer(claim, ourFinalizer), + driver: m.expectClassParameters(map[string]interface{}{className: 1}). + expectClaimParameters(map[string]interface{}{claimName: 2}). + expectAllocate(map[string]allocate{claimName: {allocErr: errors.New("fake error")}}), + expectedClaim: withFinalizer(claim, ourFinalizer), + expectedError: "allocate: fake error", + }, + // not deleted, allocated -> do nothing + "immediate-allocated-nop": { + key: claimKey, + classes: classes, + claim: withAllocate(claim), + expectedClaim: withAllocate(claim), + }, + + // not deleted, reallocate -> deallocate + "immediate-allocated-reallocate": { + key: claimKey, + classes: classes, + claim: withDeallocate(withAllocate(claim)), + driver: m.expectDeallocate(map[string]error{claimName: nil}), + expectedClaim: claim, + }, + + // not deleted, reallocate, deallocate failure -> requeue + "immediate-allocated-fail-deallocation-during-reallocate": { + key: claimKey, + classes: classes, + claim: withDeallocate(withAllocate(claim)), + driver: m.expectDeallocate(map[string]error{claimName: errors.New("fake error")}), + expectedClaim: withDeallocate(withAllocate(claim)), + expectedError: "deallocate: fake error", + }, + + // Delayed allocation is similar in some cases, but not quite + // the same. + // deletion time stamp set, our finalizer set, not allocated -> remove finalizer + "delayed-deleted-finalizer-removal": { + key: claimKey, + classes: classes, + claim: withFinalizer(withDeletionTimestamp(delayedClaim), ourFinalizer), + driver: m.expectDeallocate(map[string]error{claimName: nil}), + expectedClaim: withDeletionTimestamp(delayedClaim), + }, + // deletion time stamp set, our finalizer set, not allocated, stopping fails -> requeue + "delayed-deleted-finalizer-stop-failure": { + key: claimKey, + classes: classes, + claim: withFinalizer(withDeletionTimestamp(delayedClaim), ourFinalizer), + driver: m.expectDeallocate(map[string]error{claimName: errors.New("fake error")}), + expectedClaim: withFinalizer(withDeletionTimestamp(delayedClaim), ourFinalizer), + expectedError: "stop allocation: fake error", + }, + // deletion time stamp set, other finalizer set, not allocated -> do nothing + "delayed-deleted-finalizer-no-removal": { + key: claimKey, + classes: classes, + claim: withFinalizer(withDeletionTimestamp(delayedClaim), otherFinalizer), + expectedClaim: withFinalizer(withDeletionTimestamp(delayedClaim), otherFinalizer), + }, + // deletion time stamp set, finalizer set, allocated -> deallocate + "delayed-deleted-allocated": { + key: claimKey, + classes: classes, + claim: withAllocate(withDeletionTimestamp(delayedClaim)), + driver: m.expectDeallocate(map[string]error{claimName: nil}), + expectedClaim: withDeletionTimestamp(delayedClaim), + }, + // deletion time stamp set, finalizer set, allocated, deallocation fails -> requeue + "delayed-deleted-deallocate-failure": { + key: claimKey, + classes: classes, + claim: withAllocate(withDeletionTimestamp(delayedClaim)), + driver: m.expectDeallocate(map[string]error{claimName: errors.New("fake error")}), + expectedClaim: withAllocate(withDeletionTimestamp(delayedClaim)), + expectedError: "deallocate: fake error", + }, + // deletion time stamp set, finalizer not set -> do nothing + "delayed-deleted-no-finalizer": { + key: claimKey, + classes: classes, + claim: withDeletionTimestamp(delayedClaim), + expectedClaim: withDeletionTimestamp(delayedClaim), + }, + // waiting for first consumer -> do nothing + "delayed-pending": { + key: claimKey, + classes: classes, + claim: delayedClaim, + expectedClaim: delayedClaim, + }, + + // pod with no claims -> shouldn't occur, check again anyway + "pod-nop": { + key: podKey, + pod: pod, + podScheduling: withSelectedNode(podScheduling), + expectedPodScheduling: withSelectedNode(podScheduling), + expectedError: errPeriodic.Error(), + }, + + // pod with immediate allocation and selected node -> shouldn't occur, check again in case that claim changes + "pod-immediate": { + key: podKey, + claim: claim, + expectedClaim: claim, + pod: podWithClaim, + podScheduling: withSelectedNode(podScheduling), + expectedPodScheduling: withSelectedNode(podScheduling), + expectedError: errPeriodic.Error(), + }, + + // pod with delayed allocation, no potential nodes -> shouldn't occur + "pod-delayed-no-nodes": { + key: podKey, + classes: classes, + claim: delayedClaim, + expectedClaim: delayedClaim, + pod: podWithClaim, + podScheduling: podScheduling, + expectedPodScheduling: podScheduling, + }, + + // pod with delayed allocation, potential nodes -> provide unsuitable nodes + "pod-delayed-info": { + key: podKey, + classes: classes, + claim: delayedClaim, + expectedClaim: delayedClaim, + pod: podWithClaim, + podScheduling: withPotentialNodes(podScheduling), + driver: m.expectClassParameters(map[string]interface{}{className: 1}). + expectClaimParameters(map[string]interface{}{claimName: 2}). + expectUnsuitableNodes(map[string][]string{podClaimName: unsuitableNodes}, nil), + expectedPodScheduling: withUnsuitableNodes(withPotentialNodes(podScheduling)), + expectedError: errPeriodic.Error(), + }, + + // pod with delayed allocation, potential nodes, selected node, missing class -> failure + "pod-delayed-missing-class": { + key: podKey, + claim: delayedClaim, + expectedClaim: delayedClaim, + pod: podWithClaim, + podScheduling: withSelectedNode(withPotentialNodes(podScheduling)), + expectedPodScheduling: withSelectedNode(withPotentialNodes(podScheduling)), + expectedError: `pod claim my-pod-claim: resourceclass.resource.k8s.io "mock-class" not found`, + }, + + // pod with delayed allocation, potential nodes, selected node -> allocate + "pod-delayed-allocate": { + key: podKey, + classes: classes, + claim: delayedClaim, + expectedClaim: withReservedFor(withAllocate(delayedClaim), pod), + pod: podWithClaim, + podScheduling: withSelectedNode(withPotentialNodes(podScheduling)), + driver: m.expectClassParameters(map[string]interface{}{className: 1}). + expectClaimParameters(map[string]interface{}{claimName: 2}). + expectUnsuitableNodes(map[string][]string{podClaimName: unsuitableNodes}, nil). + expectAllocate(map[string]allocate{claimName: {allocResult: &allocation, selectedNode: nodeName, allocErr: nil}}), + expectedPodScheduling: withUnsuitableNodes(withSelectedNode(withPotentialNodes(podScheduling))), + expectedError: errPeriodic.Error(), + }, + } { + t.Run(name, func(t *testing.T) { + _, ctx := ktesting.NewTestContext(t) + initialObjects := []runtime.Object{} + for _, class := range test.classes { + initialObjects = append(initialObjects, class) + } + if test.pod != nil { + initialObjects = append(initialObjects, test.pod) + } + if test.podScheduling != nil { + initialObjects = append(initialObjects, test.podScheduling) + } + if test.claim != nil { + initialObjects = append(initialObjects, test.claim) + } + kubeClient, informerFactory := fakeK8s(initialObjects) + rcInformer := informerFactory.Resource().V1alpha1().ResourceClasses() + claimInformer := informerFactory.Resource().V1alpha1().ResourceClaims() + podInformer := informerFactory.Core().V1().Pods() + podSchedulingInformer := informerFactory.Resource().V1alpha1().PodSchedulings() + + for _, obj := range initialObjects { + switch obj.(type) { + case *resourcev1alpha1.ResourceClass: + require.NoError(t, rcInformer.Informer().GetStore().Add(obj), "add resource class") + case *resourcev1alpha1.ResourceClaim: + require.NoError(t, claimInformer.Informer().GetStore().Add(obj), "add resource claim") + case *corev1.Pod: + require.NoError(t, podInformer.Informer().GetStore().Add(obj), "add pod") + case *resourcev1alpha1.PodScheduling: + require.NoError(t, podSchedulingInformer.Informer().GetStore().Add(obj), "add pod scheduling") + default: + t.Fatalf("unknown initialObject type: %+v", obj) + } + } + + driver := test.driver + driver.t = t + + ctrl := New(ctx, driverName, driver, kubeClient, informerFactory) + _, err := ctrl.(*controller).syncKey(ctx, test.key) + if err != nil && test.expectedError == "" { + t.Fatalf("unexpected error: %v", err) + } + if err == nil && test.expectedError != "" { + t.Fatalf("did not get expected error %q", test.expectedError) + } + if err != nil && err.Error() != test.expectedError { + t.Fatalf("expected error %q, got %q", test.expectedError, err.Error()) + } + claims, err := kubeClient.ResourceV1alpha1().ResourceClaims("").List(ctx, metav1.ListOptions{}) + require.NoError(t, err, "list claims") + var expectedClaims []resourcev1alpha1.ResourceClaim + if test.expectedClaim != nil { + expectedClaims = append(expectedClaims, *test.expectedClaim) + } + assert.Equal(t, expectedClaims, claims.Items) + + podSchedulings, err := kubeClient.ResourceV1alpha1().PodSchedulings("").List(ctx, metav1.ListOptions{}) + require.NoError(t, err, "list pod schedulings") + var expectedPodSchedulings []resourcev1alpha1.PodScheduling + if test.expectedPodScheduling != nil { + expectedPodSchedulings = append(expectedPodSchedulings, *test.expectedPodScheduling) + } + assert.Equal(t, expectedPodSchedulings, podSchedulings.Items) + + // TODO: add testing of events. + // Right now, client-go/tools/record/event.go:267 fails during unit testing with + // request namespace does not match object namespace, request: "" object: "default", + }) + } +} + +type mockDriver struct { + t *testing.T + + // TODO: change this so that the mock driver expects calls in a certain order + // and fails when the next call isn't the expected one or calls didn't happen + classParameters map[string]interface{} + claimParameters map[string]interface{} + allocate map[string]allocate + deallocate map[string]error + unsuitableNodes map[string][]string + unsuitableNodesError error +} + +type allocate struct { + selectedNode string + allocResult *resourcev1alpha1.AllocationResult + allocErr error +} + +func (m mockDriver) expectClassParameters(expected map[string]interface{}) mockDriver { + m.classParameters = expected + return m +} + +func (m mockDriver) expectClaimParameters(expected map[string]interface{}) mockDriver { + m.claimParameters = expected + return m +} + +func (m mockDriver) expectAllocate(expected map[string]allocate) mockDriver { + m.allocate = expected + return m +} + +func (m mockDriver) expectDeallocate(expected map[string]error) mockDriver { + m.deallocate = expected + return m +} + +func (m mockDriver) expectUnsuitableNodes(expected map[string][]string, err error) mockDriver { + m.unsuitableNodes = expected + m.unsuitableNodesError = err + return m +} + +func (m mockDriver) GetClassParameters(ctx context.Context, class *resourcev1alpha1.ResourceClass) (interface{}, error) { + m.t.Logf("GetClassParameters(%s)", class) + result, ok := m.classParameters[class.Name] + if !ok { + m.t.Fatal("unexpected GetClassParameters call") + } + if err, ok := result.(error); ok { + return nil, err + } + return result, nil +} + +func (m mockDriver) GetClaimParameters(ctx context.Context, claim *resourcev1alpha1.ResourceClaim, class *resourcev1alpha1.ResourceClass, classParameters interface{}) (interface{}, error) { + m.t.Logf("GetClaimParameters(%s)", claim) + result, ok := m.claimParameters[claim.Name] + if !ok { + m.t.Fatal("unexpected GetClaimParameters call") + } + if err, ok := result.(error); ok { + return nil, err + } + return result, nil +} + +func (m mockDriver) Allocate(ctx context.Context, claim *resourcev1alpha1.ResourceClaim, claimParameters interface{}, class *resourcev1alpha1.ResourceClass, classParameters interface{}, selectedNode string) (*resourcev1alpha1.AllocationResult, error) { + m.t.Logf("Allocate(%s)", claim) + allocate, ok := m.allocate[claim.Name] + if !ok { + m.t.Fatal("unexpected Allocate call") + } + assert.Equal(m.t, allocate.selectedNode, selectedNode, "selected node") + return allocate.allocResult, allocate.allocErr +} + +func (m mockDriver) Deallocate(ctx context.Context, claim *resourcev1alpha1.ResourceClaim) error { + m.t.Logf("Deallocate(%s)", claim) + err, ok := m.deallocate[claim.Name] + if !ok { + m.t.Fatal("unexpected Deallocate call") + } + return err +} + +func (m mockDriver) UnsuitableNodes(ctx context.Context, pod *corev1.Pod, claims []*ClaimAllocation, potentialNodes []string) error { + m.t.Logf("UnsuitableNodes(%s, %v, %v)", pod, claims, potentialNodes) + if len(m.unsuitableNodes) == 0 { + m.t.Fatal("unexpected UnsuitableNodes call") + } + if m.unsuitableNodesError != nil { + return m.unsuitableNodesError + } + found := map[string]bool{} + for _, delayed := range claims { + unsuitableNodes, ok := m.unsuitableNodes[delayed.PodClaimName] + if !ok { + m.t.Errorf("unexpected pod claim: %s", delayed.PodClaimName) + } + delayed.UnsuitableNodes = unsuitableNodes + found[delayed.PodClaimName] = true + } + for expectedName := range m.unsuitableNodes { + if !found[expectedName] { + m.t.Errorf("pod claim %s not in actual claims list", expectedName) + } + } + return nil +} + +func createClass(className, driverName string) *resourcev1alpha1.ResourceClass { + return &resourcev1alpha1.ResourceClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: className, + }, + DriverName: driverName, + } +} + +func createClaim(claimName, claimNamespace, className string) *resourcev1alpha1.ResourceClaim { + return &resourcev1alpha1.ResourceClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: claimName, + Namespace: claimNamespace, + }, + Spec: resourcev1alpha1.ResourceClaimSpec{ + ResourceClassName: className, + AllocationMode: resourcev1alpha1.AllocationModeImmediate, + }, + } +} + +func createPod(podName, podNamespace string, claims map[string]string) *corev1.Pod { + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: podName, + Namespace: podNamespace, + UID: "1234", + }, + } + for podClaimName, claimName := range claims { + pod.Spec.ResourceClaims = append(pod.Spec.ResourceClaims, + corev1.PodResourceClaim{ + Name: podClaimName, + Source: corev1.ClaimSource{ + ResourceClaimName: &claimName, + }, + }, + ) + } + return pod +} + +func createPodScheduling(pod *corev1.Pod) *resourcev1alpha1.PodScheduling { + controller := true + return &resourcev1alpha1.PodScheduling{ + ObjectMeta: metav1.ObjectMeta{ + Name: pod.Name, + Namespace: pod.Namespace, + OwnerReferences: []metav1.OwnerReference{ + { + Name: pod.Name, + Controller: &controller, + UID: pod.UID, + }, + }, + }, + } +} + +func fakeK8s(objs []runtime.Object) (kubernetes.Interface, informers.SharedInformerFactory) { + // This is a very simple replacement for a real apiserver. For example, + // it doesn't do defaulting and accepts updates to the status in normal + // Update calls. Therefore this test does not catch when we use Update + // instead of UpdateStatus. Reactors could be used to catch that, but + // that seems overkill because E2E tests will find that. + // + // Interactions with the fake apiserver also never fail. TODO: + // simulate update errors. + client := fake.NewSimpleClientset(objs...) + informerFactory := informers.NewSharedInformerFactory(client, 0) + return client, informerFactory +} diff --git a/staging/src/k8s.io/dynamic-resource-allocation/go.mod b/staging/src/k8s.io/dynamic-resource-allocation/go.mod index c84d68fe454..dd039c2bdbf 100644 --- a/staging/src/k8s.io/dynamic-resource-allocation/go.mod +++ b/staging/src/k8s.io/dynamic-resource-allocation/go.mod @@ -5,29 +5,61 @@ module k8s.io/dynamic-resource-allocation go 1.19 require ( + github.com/go-logr/logr v1.2.3 + github.com/google/go-cmp v0.5.9 + github.com/stretchr/testify v1.8.0 + google.golang.org/grpc v1.49.0 k8s.io/api v0.0.0 k8s.io/apimachinery v0.0.0 + k8s.io/client-go v0.0.0 + k8s.io/klog/v2 v2.80.1 + k8s.io/kubelet v0.0.0 ) require ( - github.com/go-logr/logr v1.2.3 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/emicklei/go-restful/v3 v3.9.0 // indirect + github.com/evanphx/json-patch v4.12.0+incompatible // indirect + github.com/go-openapi/jsonpointer v0.19.5 // indirect + github.com/go-openapi/jsonreference v0.20.0 // indirect + github.com/go-openapi/swag v0.19.14 // indirect github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/protobuf v1.5.2 // indirect + github.com/google/gnostic v0.5.7-v3refs // indirect github.com/google/gofuzz v1.1.0 // indirect + github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/mailru/easyjson v0.7.6 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect golang.org/x/net v0.1.1-0.20221027164007-c63010009c80 // indirect + golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b // indirect + golang.org/x/sys v0.1.0 // indirect + golang.org/x/term v0.1.0 // indirect golang.org/x/text v0.4.0 // indirect + golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect + google.golang.org/appengine v1.6.7 // indirect + google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 // indirect + google.golang.org/protobuf v1.28.1 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect - k8s.io/klog/v2 v2.80.1 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 // indirect k8s.io/utils v0.0.0-20221107191617-1a15be271d1d // indirect sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect + sigs.k8s.io/yaml v1.3.0 // indirect ) replace ( k8s.io/api => ../api k8s.io/apimachinery => ../apimachinery + k8s.io/client-go => ../client-go + k8s.io/component-base => ../component-base k8s.io/dynamic-resource-allocation => ../dynamic-resource-allocation + k8s.io/kubelet => ../kubelet ) diff --git a/staging/src/k8s.io/dynamic-resource-allocation/go.sum b/staging/src/k8s.io/dynamic-resource-allocation/go.sum index 75b1d79a590..6211f340ff7 100644 --- a/staging/src/k8s.io/dynamic-resource-allocation/go.sum +++ b/staging/src/k8s.io/dynamic-resource-allocation/go.sum @@ -1,77 +1,513 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= +github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= +github.com/emicklei/go-restful/v3 v3.9.0 h1:XwGDlfxEnQZzuopoqxwSEllNcCOM9DhhFyhFIIGKwxE= +github.com/emicklei/go-restful/v3 v3.9.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= +github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= +github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonreference v0.20.0 h1:MYlu0sBgChmCfJxxUKZ8g1cPWFOB37YSZqewK7OKeyA= +github.com/go-openapi/jsonreference v0.20.0/go.mod h1:Ag74Ico3lPc+zR+qjn4XBUmXymS4zJbYVCZmcgkasdo= +github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/swag v0.19.14 h1:gm3vOOXfiuw5i9p5N9xJvfjvuofpyvLA9Wr6QfK5Fng= +github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/gnostic v0.5.7-v3refs h1:FhTMOKj2VhjpouxvWJAV1TL304uMlb9zcDqkl6cEI54= +github.com/google/gnostic v0.5.7-v3refs/go.mod h1:73MKFl6jIHelAJNaBGFzt3SPtZULs9dYrGFt8OiIsHQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA= +github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/onsi/ginkgo/v2 v2.4.0 h1:+Ig9nvqgS5OBSACXNk15PLdp0U9XPYROt9CFzVdFGIs= +github.com/onsi/gomega v1.23.0 h1:/oxKu9c2HVap+F3PfKort2Hw5DEU+HGlW8n+tguWsys= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.1.1-0.20221027164007-c63010009c80 h1:CtRWmqbiPSOXwJV1JoY7pWiTx2xzVKQ813bvU+Y/9jI= golang.org/x/net v0.1.1-0.20221027164007-c63010009c80/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b h1:clP8eMhB30EHdc0bd2Twtq6kgU7yl5ub2cQLSdrv1Dg= +golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.1.0 h1:g6Z6vPFA9dYBAF7DWcH6sCcOntplXsDKcliusYijMlw= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 h1:vVKdlvoWBphwdxWKrFZEuM0kGgGLxUOYcY4U/2Vjg44= +golang.org/x/time v0.0.0-20220210224613-90d013bbcef8/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 h1:hrbNEivu7Zn1pxvHk6MBrq9iE22woVILTHqexqBxe6I= +google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= +google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= +google.golang.org/grpc v1.49.0 h1:WTLtQzmQori5FUH25Pq4WT22oCsv8USpQ+F6rqtsmxw= +google.golang.org/grpc v1.49.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= +google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= k8s.io/klog/v2 v2.80.1 h1:atnLQ121W371wYYFawwYx1aEY2eUfs4l3J72wtgAwV4= k8s.io/klog/v2 v2.80.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 h1:+70TFaan3hfJzs+7VK2o+OGxg8HsuBr/5f6tVAjDu6E= +k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280/go.mod h1:+Axhij7bCpeqhklhUTe3xmOn6bWxolyZEeyaFpjGtl4= k8s.io/utils v0.0.0-20221107191617-1a15be271d1d h1:0Smp/HP1OH4Rvhe+4B8nWGERtlqAGSftbSbbmm45oFs= k8s.io/utils v0.0.0-20221107191617-1a15be271d1d/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 h1:iXTIw73aPyC+oRdyqqvVJuloN1p0AC/kzH07hu3NE+k= sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= +sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= diff --git a/staging/src/k8s.io/dynamic-resource-allocation/kubeletplugin/doc.go b/staging/src/k8s.io/dynamic-resource-allocation/kubeletplugin/doc.go new file mode 100644 index 00000000000..cdc33100a3f --- /dev/null +++ b/staging/src/k8s.io/dynamic-resource-allocation/kubeletplugin/doc.go @@ -0,0 +1,19 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package kubeletplugin provides helper functions for running a dynamic +// resource allocation kubelet plugin. +package kubeletplugin diff --git a/staging/src/k8s.io/dynamic-resource-allocation/kubeletplugin/draplugin.go b/staging/src/k8s.io/dynamic-resource-allocation/kubeletplugin/draplugin.go new file mode 100644 index 00000000000..cfe1428b9f3 --- /dev/null +++ b/staging/src/k8s.io/dynamic-resource-allocation/kubeletplugin/draplugin.go @@ -0,0 +1,236 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubeletplugin + +import ( + "errors" + "fmt" + "net" + + "google.golang.org/grpc" + "k8s.io/klog/v2" + + drapbv1 "k8s.io/kubelet/pkg/apis/dra/v1alpha1" + registerapi "k8s.io/kubelet/pkg/apis/pluginregistration/v1" +) + +// DRAPlugin gets returned by Start and defines the public API of the generic +// dynamic resource allocation plugin. +type DRAPlugin interface { + // Stop ensures that all spawned goroutines are stopped and frees + // resources. + Stop() + + // RegistrationStatus returns the result of registration, nil if none + // received yet. + RegistrationStatus() *registerapi.RegistrationStatus + + // This unexported method ensures that we can modify the interface + // without causing an API break of the package + // (https://pkg.go.dev/golang.org/x/exp/apidiff#section-readme). + internal() +} + +// Option implements the functional options pattern for Start. +type Option func(o *options) error + +// DriverName defines the driver name for the dynamic resource allocation driver. +// Must be set. +func DriverName(driverName string) Option { + return func(o *options) error { + o.driverName = driverName + return nil + } +} + +// Logger overrides the default klog.Background logger. +func Logger(logger klog.Logger) Option { + return func(o *options) error { + o.logger = logger + return nil + } +} + +// GRPCVerbosity sets the verbosity for logging gRPC calls. Default is 4. A negative +// value disables logging. +func GRPCVerbosity(level int) Option { + return func(o *options) error { + o.grpcVerbosity = level + return nil + } +} + +// RegistrarSocketPath sets the file path for a Unix domain socket. +// If RegistrarListener is not used, then Start will remove +// a file at that path, should one exist, and creates a socket +// itself. Otherwise it uses the provided listener and only +// removes the socket at the specified path during shutdown. +// +// At least one of these two options is required. +func RegistrarSocketPath(path string) Option { + return func(o *options) error { + o.pluginRegistrationEndpoint.path = path + return nil + } +} + +// RegistrarListener sets an already created listener for the plugin +// registrarion API. Can be combined with RegistrarSocketPath. +// +// At least one of these two options is required. +func RegistrarListener(listener net.Listener) Option { + return func(o *options) error { + o.pluginRegistrationEndpoint.listener = listener + return nil + } +} + +// PluginSocketPath sets the file path for a Unix domain socket. +// If PluginListener is not used, then Start will remove +// a file at that path, should one exist, and creates a socket +// itself. Otherwise it uses the provided listener and only +// removes the socket at the specified path during shutdown. +// +// At least one of these two options is required. +func PluginSocketPath(path string) Option { + return func(o *options) error { + o.draEndpoint.path = path + return nil + } +} + +// PluginListener sets an already created listener for the dynamic resource +// allocation plugin API. Can be combined with PluginSocketPath. +// +// At least one of these two options is required. +func PluginListener(listener net.Listener) Option { + return func(o *options) error { + o.draEndpoint.listener = listener + return nil + } +} + +// KubeletPluginSocketPath defines how kubelet will connect to the dynamic +// resource allocation plugin. This corresponds to PluginSocketPath, except +// that PluginSocketPath defines the path in the filesystem of the caller and +// KubeletPluginSocketPath in the filesystem of kubelet. +func KubeletPluginSocketPath(path string) Option { + return func(o *options) error { + o.draAddress = path + return nil + } +} + +// GRPCInterceptor is called for each incoming gRPC method call. +func GRPCInterceptor(interceptor grpc.UnaryServerInterceptor) Option { + return func(o *options) error { + o.interceptor = interceptor + return nil + } +} + +type options struct { + logger klog.Logger + grpcVerbosity int + driverName string + draEndpoint endpoint + draAddress string + pluginRegistrationEndpoint endpoint + interceptor grpc.UnaryServerInterceptor +} + +// draPlugin combines the kubelet registration service and the DRA node plugin +// service. +type draPlugin struct { + registrar *nodeRegistrar + plugin *grpcServer +} + +// Start sets up two gRPC servers (one for registration, one for the DRA node +// client). +func Start(nodeServer drapbv1.NodeServer, opts ...Option) (result DRAPlugin, finalErr error) { + d := &draPlugin{} + + o := options{ + logger: klog.Background(), + grpcVerbosity: 4, + } + for _, option := range opts { + if err := option(&o); err != nil { + return nil, err + } + } + + if o.driverName == "" { + return nil, errors.New("driver name must be set") + } + if o.draAddress == "" { + return nil, errors.New("DRA address must be set") + } + var emptyEndpoint endpoint + if o.draEndpoint == emptyEndpoint { + return nil, errors.New("a Unix domain socket path and/or listener must be set for the kubelet plugin") + } + if o.pluginRegistrationEndpoint == emptyEndpoint { + return nil, errors.New("a Unix domain socket path and/or listener must be set for the registrar") + } + + // Run the node plugin gRPC server first to ensure that it is ready. + plugin, err := startGRPCServer(klog.LoggerWithName(o.logger, "dra"), o.grpcVerbosity, o.interceptor, o.draEndpoint, func(grpcServer *grpc.Server) { + drapbv1.RegisterNodeServer(grpcServer, nodeServer) + }) + if err != nil { + return nil, fmt.Errorf("start node client: %v", err) + } + d.plugin = plugin + defer func() { + // Clean up if we didn't finish succcessfully. + if r := recover(); r != nil { + plugin.stop() + panic(r) + } + if finalErr != nil { + plugin.stop() + } + }() + + // Now make it available to kubelet. + registrar, err := startRegistrar(klog.LoggerWithName(o.logger, "registrar"), o.grpcVerbosity, o.interceptor, o.driverName, o.draAddress, o.pluginRegistrationEndpoint) + if err != nil { + return nil, fmt.Errorf("start registrar: %v", err) + } + d.registrar = registrar + + return d, nil +} + +func (d *draPlugin) Stop() { + if d == nil { + return + } + d.registrar.stop() + d.plugin.stop() +} + +func (d *draPlugin) RegistrationStatus() *registerapi.RegistrationStatus { + if d.registrar == nil { + return nil + } + return d.registrar.status +} + +func (d *draPlugin) internal() {} diff --git a/staging/src/k8s.io/dynamic-resource-allocation/kubeletplugin/noderegistrar.go b/staging/src/k8s.io/dynamic-resource-allocation/kubeletplugin/noderegistrar.go new file mode 100644 index 00000000000..1f37db22fe9 --- /dev/null +++ b/staging/src/k8s.io/dynamic-resource-allocation/kubeletplugin/noderegistrar.go @@ -0,0 +1,60 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubeletplugin + +import ( + "fmt" + + "google.golang.org/grpc" + "k8s.io/klog/v2" + registerapi "k8s.io/kubelet/pkg/apis/pluginregistration/v1" +) + +type nodeRegistrar struct { + logger klog.Logger + registrationServer + server *grpcServer +} + +// startRegistrar returns a running instance. +func startRegistrar(logger klog.Logger, grpcVerbosity int, interceptor grpc.UnaryServerInterceptor, driverName string, endpoint string, pluginRegistrationEndpoint endpoint) (*nodeRegistrar, error) { + n := &nodeRegistrar{ + logger: logger, + registrationServer: registrationServer{ + driverName: driverName, + endpoint: endpoint, + supportedVersions: []string{"1.0.0"}, // TODO: is this correct? + }, + } + s, err := startGRPCServer(logger, grpcVerbosity, interceptor, pluginRegistrationEndpoint, func(grpcServer *grpc.Server) { + registerapi.RegisterRegistrationServer(grpcServer, n) + }) + if err != nil { + return nil, fmt.Errorf("start gRPC server: %v", err) + } + n.server = s + return n, nil +} + +// stop ensures that the registrar is not running anymore and cleans up all resources. +// It is idempotent and may be called with a nil pointer. +func (s *nodeRegistrar) stop() { + if s == nil { + return + } + s.server.stop() +} diff --git a/staging/src/k8s.io/dynamic-resource-allocation/kubeletplugin/nonblockinggrpcserver.go b/staging/src/k8s.io/dynamic-resource-allocation/kubeletplugin/nonblockinggrpcserver.go new file mode 100644 index 00000000000..16df39710de --- /dev/null +++ b/staging/src/k8s.io/dynamic-resource-allocation/kubeletplugin/nonblockinggrpcserver.go @@ -0,0 +1,151 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubeletplugin + +import ( + "context" + "fmt" + "net" + "os" + "sync" + "sync/atomic" + + "google.golang.org/grpc" + "k8s.io/klog/v2" +) + +type grpcServer struct { + logger klog.Logger + grpcVerbosity int + wg sync.WaitGroup + endpoint endpoint + server *grpc.Server + requestID int64 +} + +type registerService func(s *grpc.Server) + +// endpoint defines where to listen for incoming connections. +// The listener always gets closed when shutting down. +// +// If the listener is not set, a new listener for a Unix domain socket gets +// created at the path. +// +// If the path is non-empty, then the socket will get removed when shutting +// down, regardless of who created the listener. +type endpoint struct { + path string + listener net.Listener +} + +// startGRPCServer sets up the GRPC server on a Unix domain socket and spawns a goroutine +// which handles requests for arbitrary services. +func startGRPCServer(logger klog.Logger, grpcVerbosity int, interceptor grpc.UnaryServerInterceptor, endpoint endpoint, services ...registerService) (*grpcServer, error) { + s := &grpcServer{ + logger: logger, + endpoint: endpoint, + grpcVerbosity: grpcVerbosity, + } + + listener := endpoint.listener + if listener == nil { + // Remove any (probably stale) existing socket. + if err := os.Remove(endpoint.path); err != nil && !os.IsNotExist(err) { + return nil, fmt.Errorf("remove Unix domain socket: %v", err) + } + + // Now we can use the endpoint for listening. + l, err := net.Listen("unix", endpoint.path) + if err != nil { + return nil, fmt.Errorf("listen on %q: %v", endpoint.path, err) + } + listener = l + } + + // Run a gRPC server. It will close the listening socket when + // shutting down, so we don't need to do that. + var opts []grpc.ServerOption + var interceptors []grpc.UnaryServerInterceptor + if grpcVerbosity >= 0 { + interceptors = append(interceptors, s.interceptor) + } + if interceptor != nil { + interceptors = append(interceptors, interceptor) + } + if len(interceptors) >= 0 { + opts = append(opts, grpc.ChainUnaryInterceptor(interceptors...)) + } + s.server = grpc.NewServer(opts...) + for _, service := range services { + service(s.server) + } + s.wg.Add(1) + go func() { + defer s.wg.Done() + err := s.server.Serve(listener) + if err != nil { + logger.Error(err, "GRPC server failed") + } else { + logger.V(3).Info("GRPC server terminated gracefully") + } + }() + + logger.Info("GRPC server started") + return s, nil +} + +// interceptor is called for each request. It creates a logger with a unique, +// sequentially increasing request ID and adds that logger to the context. It +// also logs request and response. +func (s *grpcServer) interceptor(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp interface{}, err error) { + requestID := atomic.AddInt64(&s.requestID, 1) + logger := klog.LoggerWithValues(s.logger, "requestID", requestID) + ctx = klog.NewContext(ctx, logger) + logger.V(s.grpcVerbosity).Info("handling request", "request", req) + defer func() { + if r := recover(); r != nil { + logger.Error(nil, "handling request panicked", "panic", r, "request", req) + panic(r) + } + }() + resp, err = handler(ctx, req) + if err != nil { + logger.Error(err, "handling request failed", "request", req) + } else { + logger.V(s.grpcVerbosity).Info("handling request succeeded", "response", resp) + } + return +} + +// stop ensures that the server is not running anymore and cleans up all resources. +// It is idempotent and may be called with a nil pointer. +func (s *grpcServer) stop() { + if s == nil { + return + } + if s.server != nil { + s.server.Stop() + } + s.wg.Wait() + s.server = nil + if s.endpoint.path != "" { + if err := os.Remove(s.endpoint.path); err != nil && !os.IsNotExist(err) { + s.logger.Error(err, "remove Unix socket") + } + } + s.logger.V(3).Info("GRPC server stopped") +} diff --git a/staging/src/k8s.io/dynamic-resource-allocation/kubeletplugin/registrationserver.go b/staging/src/k8s.io/dynamic-resource-allocation/kubeletplugin/registrationserver.go new file mode 100644 index 00000000000..4f0adefb303 --- /dev/null +++ b/staging/src/k8s.io/dynamic-resource-allocation/kubeletplugin/registrationserver.go @@ -0,0 +1,54 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubeletplugin + +import ( + "context" + "fmt" + + registerapi "k8s.io/kubelet/pkg/apis/pluginregistration/v1" +) + +// registrationServer implements the kubelet plugin registration gRPC interface. +type registrationServer struct { + driverName string + endpoint string + supportedVersions []string + status *registerapi.RegistrationStatus +} + +var _ registerapi.RegistrationServer = ®istrationServer{} + +// GetInfo is the RPC invoked by plugin watcher. +func (e *registrationServer) GetInfo(ctx context.Context, req *registerapi.InfoRequest) (*registerapi.PluginInfo, error) { + return ®isterapi.PluginInfo{ + Type: registerapi.DRAPlugin, + Name: e.driverName, + Endpoint: e.endpoint, + SupportedVersions: e.supportedVersions, + }, nil +} + +// NotifyRegistrationStatus is the RPC invoked by plugin watcher. +func (e *registrationServer) NotifyRegistrationStatus(ctx context.Context, status *registerapi.RegistrationStatus) (*registerapi.RegistrationStatusResponse, error) { + e.status = status + if !status.PluginRegistered { + return nil, fmt.Errorf("failed registration process: %+v", status.Error) + } + + return ®isterapi.RegistrationStatusResponse{}, nil +} diff --git a/staging/src/k8s.io/dynamic-resource-allocation/leaderelection/leaderelection.go b/staging/src/k8s.io/dynamic-resource-allocation/leaderelection/leaderelection.go new file mode 100644 index 00000000000..d6e031e4792 --- /dev/null +++ b/staging/src/k8s.io/dynamic-resource-allocation/leaderelection/leaderelection.go @@ -0,0 +1,262 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package leaderelection wraps k8s.io/client-go/tools/leaderelection with a +// simpler API. It's derived from https://github.com/kubernetes-csi/csi-lib-utils/tree/v0.11.0/leaderelection +package leaderelection + +import ( + "context" + "fmt" + "net/http" + "os" + "regexp" + "strings" + "time" + + v1 "k8s.io/api/core/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/scheme" + corev1 "k8s.io/client-go/kubernetes/typed/core/v1" + "k8s.io/client-go/tools/leaderelection" + "k8s.io/client-go/tools/leaderelection/resourcelock" + "k8s.io/client-go/tools/record" + "k8s.io/klog/v2" +) + +const ( + defaultLeaseDuration = 15 * time.Second + defaultRenewDeadline = 10 * time.Second + defaultRetryPeriod = 5 * time.Second + defaultHealthCheckTimeout = 20 * time.Second + + // HealthCheckerAddress is the address at which the leader election health + // checker reports status. + // The caller sidecar should document this address in appropriate flag + // descriptions. + HealthCheckerAddress = "/healthz/leader-election" +) + +// leaderElection is a convenience wrapper around client-go's leader election library. +type leaderElection struct { + runFunc func(ctx context.Context) + + // the lockName identifies the leader election config and should be shared across all members + lockName string + // the identity is the unique identity of the currently running member + identity string + // the namespace to store the lock resource + namespace string + // resourceLock defines the type of leaderelection that should be used + // Only resourcelock.LeasesResourceLock is valid at the moment. + resourceLock string + // healthCheck reports unhealthy if leader election fails to renew leadership + // within a timeout period. + healthCheck *leaderelection.HealthzAdaptor + + leaseDuration time.Duration + renewDeadline time.Duration + retryPeriod time.Duration + healthCheckTimeout time.Duration + + ctx context.Context + + clientset kubernetes.Interface +} + +// Option implements functional options for New. +type Option func(l *leaderElection) + +// New constructs a new leader election instance. +func New(clientset kubernetes.Interface, lockName string, runFunc func(ctx context.Context), opts ...Option) *leaderElection { + l := &leaderElection{ + runFunc: runFunc, + lockName: lockName, + resourceLock: resourcelock.LeasesResourceLock, + leaseDuration: defaultLeaseDuration, + renewDeadline: defaultRenewDeadline, + retryPeriod: defaultRetryPeriod, + healthCheckTimeout: defaultHealthCheckTimeout, + clientset: clientset, + } + + for _, opt := range opts { + opt(l) + } + + return l +} + +func Identity(identity string) Option { + return func(l *leaderElection) { + l.identity = identity + } +} + +func Namespace(namespace string) Option { + return func(l *leaderElection) { + l.namespace = namespace + } +} + +func LeaseDuration(leaseDuration time.Duration) Option { + return func(l *leaderElection) { + l.leaseDuration = leaseDuration + } +} + +func RenewDeadline(renewDeadline time.Duration) Option { + return func(l *leaderElection) { + l.renewDeadline = renewDeadline + } +} + +func RetryPeriod(retryPeriod time.Duration) Option { + return func(l *leaderElection) { + l.retryPeriod = retryPeriod + } +} + +func HealthCheckTimeout(timeout time.Duration) Option { + return func(l *leaderElection) { + l.healthCheckTimeout = timeout + } +} + +func Context(ctx context.Context) Option { + return func(l *leaderElection) { + l.ctx = ctx + } +} + +// Server represents any type that could serve HTTP requests for the leader +// election health check endpoint. +type Server interface { + Handle(pattern string, handler http.Handler) +} + +// PrepareHealthCheck creates a health check for this leader election object +// with the given healthCheckTimeout and registers its HTTP handler to the given +// server at the path specified by the constant "healthCheckerAddress". +// healthCheckTimeout determines the max duration beyond lease expiration +// allowed before reporting unhealthy. +// The caller sidecar should document the handler address in appropriate flag +// descriptions. +func (l *leaderElection) PrepareHealthCheck(s Server) { + l.healthCheck = leaderelection.NewLeaderHealthzAdaptor(l.healthCheckTimeout) + s.Handle(HealthCheckerAddress, adaptCheckToHandler(l.healthCheck.Check)) +} + +func (l *leaderElection) Run() error { + if l.identity == "" { + id, err := defaultLeaderElectionIdentity() + if err != nil { + return fmt.Errorf("error getting the default leader identity: %v", err) + } + + l.identity = id + } + + if l.namespace == "" { + l.namespace = inClusterNamespace() + } + + broadcaster := record.NewBroadcaster() + broadcaster.StartRecordingToSink(&corev1.EventSinkImpl{Interface: l.clientset.CoreV1().Events(l.namespace)}) + eventRecorder := broadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: fmt.Sprintf("%s/%s", l.lockName, l.identity)}) + + rlConfig := resourcelock.ResourceLockConfig{ + Identity: sanitizeName(l.identity), + EventRecorder: eventRecorder, + } + + lock, err := resourcelock.New(l.resourceLock, l.namespace, sanitizeName(l.lockName), l.clientset.CoreV1(), l.clientset.CoordinationV1(), rlConfig) + if err != nil { + return err + } + + ctx := l.ctx + if ctx == nil { + ctx = context.Background() + } + leaderConfig := leaderelection.LeaderElectionConfig{ + Lock: lock, + LeaseDuration: l.leaseDuration, + RenewDeadline: l.renewDeadline, + RetryPeriod: l.retryPeriod, + Callbacks: leaderelection.LeaderCallbacks{ + OnStartedLeading: func(ctx context.Context) { + klog.FromContext(ctx).Info("became leader, starting") + l.runFunc(ctx) + }, + OnStoppedLeading: func() { + klog.FromContext(ctx).Error(nil, "stopped leading") + klog.FlushAndExit(klog.ExitFlushTimeout, 1) + }, + OnNewLeader: func(identity string) { + klog.FromContext(ctx).Info("new leader detected", "idendity", identity) + }, + }, + WatchDog: l.healthCheck, + } + + leaderelection.RunOrDie(ctx, leaderConfig) + return nil // should never reach here +} + +func defaultLeaderElectionIdentity() (string, error) { + return os.Hostname() +} + +// sanitizeName sanitizes the provided string so it can be consumed by leader election library +func sanitizeName(name string) string { + re := regexp.MustCompile("[^a-zA-Z0-9-]") + name = re.ReplaceAllString(name, "-") + if name[len(name)-1] == '-' { + // name must not end with '-' + name = name + "X" + } + return name +} + +// inClusterNamespace returns the namespace in which the pod is running in by checking +// the env var POD_NAMESPACE, then the file /var/run/secrets/kubernetes.io/serviceaccount/namespace. +// if neither returns a valid namespace, the "default" namespace is returned +func inClusterNamespace() string { + if ns := os.Getenv("POD_NAMESPACE"); ns != "" { + return ns + } + + if data, err := os.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/namespace"); err == nil { + if ns := strings.TrimSpace(string(data)); len(ns) > 0 { + return ns + } + } + + return "default" +} + +// adaptCheckToHandler returns an http.HandlerFunc that serves the provided checks. +func adaptCheckToHandler(c func(r *http.Request) error) http.HandlerFunc { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + err := c(r) + if err != nil { + http.Error(w, fmt.Sprintf("internal server error: %v", err), http.StatusInternalServerError) + } else { + fmt.Fprint(w, "ok") + } + }) +}