DRA resourceslice controller: support publishing multiple slices

The driver determines what each slice is meant to look like. The controller
then ensures that only those slices exist. It reuses existing slices where the
set of devices, as identified by their names, is the same as in some desired
slice. Such slices get updated to match the desired state.

In other words, attributes and the order of devices can be changed by updating
an existing slice, but adding or removing a device is done by deleting and
re-creating slices.

Co-authored-by: googs1025 <googs1025@gmail.com>

The test update is partly based on
https://github.com/kubernetes/kubernetes/pull/127645.
This commit is contained in:
Patrick Ohly
2024-10-22 22:26:03 +02:00
parent 8b063a6a08
commit 26650371cc
5 changed files with 1228 additions and 275 deletions

View File

@@ -0,0 +1,247 @@
/*
Copyright 2024 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package workqueue
import (
"maps"
"slices"
"sync"
"time"
"k8s.io/client-go/util/workqueue"
)
// TODO (pohly): move this to k8s.io/client-go/util/workqueue/mockqueue.go
// if it turns out to be generally useful. Doc comments are already written
// as if the code was there.
// MockQueue is an implementation of [TypedRateLimitingInterface] which
// can be used to test a function which pulls work items out of a queue
// and processes them. It is thread-safe.
//
// A null instance is directly usable. The usual usage is:
//
// var m workqueue.Mock[string]
// m.SyncOne("some-item", func(queue workqueue.TypedRateLimitingInterface[string]) { ... } )
// if diff := cmp.Diff(workqueue.MockState[string]{}, m.State()); diff != "" {
// t.Errorf("unexpected state of mock work queue after sync (-want, +got):\n%s", diff)
// }
//
// All slices get reset to nil when they become empty, so there are no spurious
// differences because of nil vs. empty slice.
type Mock[T comparable] struct {
mutex sync.Mutex
state MockState[T]
}
type MockState[T comparable] struct {
// Ready contains the items which are ready for processing.
Ready []T
// InFlight contains the items which are currently being processed (= Get
// was called, Done not yet).
InFlight []T
// MismatchedDone contains the items for which Done was called without
// a matching Get.
MismatchedDone []T
// Later contains the items which are meant to be added to the queue after
// a certain delay (= AddAfter was called for them). They appear in the
// order in which AddAfter got called.
Later []MockDelayedItem[T]
// Failures contains the items and their retry count which failed to be
// processed (AddRateLimited called at least once, Forget not yet).
// The retry count is always larger than zero.
Failures map[T]int
// ShutDownCalled tracks how often ShutDown got called.
ShutDownCalled int
// ShutDownWithDrainCalled tracks how often ShutDownWithDrain got called.
ShutDownWithDrainCalled int
}
// DeepCopy takes a snapshot of all slices. It cannot do a deep copy of the items in those slices,
// but typically those keys are immutable.
func (m MockState[T]) DeepCopy() *MockState[T] {
m.Ready = slices.Clone(m.Ready)
m.InFlight = slices.Clone(m.InFlight)
m.MismatchedDone = slices.Clone(m.MismatchedDone)
m.Later = slices.Clone(m.Later)
m.Failures = maps.Clone(m.Failures)
return &m
}
// MockDelayedItem is an item which was queue for later processing.
type MockDelayedItem[T comparable] struct {
Item T
Duration time.Duration
}
// SyncOne adds the item to the work queue and calls sync.
// That sync function can pull one or more items from the work
// queue until the queue is empty. Then it is told that the queue
// is shutting down, which must cause it to return.
//
// The test can then retrieve the state of the queue to check the result.
func (m *Mock[T]) SyncOne(item T, sync func(workqueue.TypedRateLimitingInterface[T])) {
// sync must run with the mutex not locked.
defer sync(m)
m.mutex.Lock()
defer m.mutex.Unlock()
m.state.Ready = append(m.state.Ready, item)
}
// State returns the current state of the queue.
func (m *Mock[T]) State() MockState[T] {
m.mutex.Lock()
defer m.mutex.Unlock()
return *m.state.DeepCopy()
}
// Add implements [TypedInterface].
func (m *Mock[T]) Add(item T) {
m.mutex.Lock()
defer m.mutex.Unlock()
if !slices.Contains(m.state.Ready, item) {
m.state.Ready = append(m.state.Ready, item)
}
}
// Len implements [TypedInterface].
func (m *Mock[T]) Len() int {
m.mutex.Lock()
defer m.mutex.Unlock()
return len(m.state.Ready)
}
// Get implements [TypedInterface].
func (m *Mock[T]) Get() (item T, shutdown bool) {
m.mutex.Lock()
defer m.mutex.Unlock()
if len(m.state.Ready) == 0 {
shutdown = true
return
}
item = m.state.Ready[0]
m.state.Ready = m.state.Ready[1:]
if len(m.state.Ready) == 0 {
m.state.Ready = nil
}
m.state.InFlight = append(m.state.InFlight, item)
return item, false
}
// Done implements [TypedInterface].
func (m *Mock[T]) Done(item T) {
m.mutex.Lock()
defer m.mutex.Unlock()
index := slices.Index(m.state.InFlight, item)
if index < 0 {
m.state.MismatchedDone = append(m.state.MismatchedDone, item)
}
m.state.InFlight = slices.Delete(m.state.InFlight, index, index+1)
if len(m.state.InFlight) == 0 {
m.state.InFlight = nil
}
}
// ShutDown implements [TypedInterface].
func (m *Mock[T]) ShutDown() {
m.mutex.Lock()
defer m.mutex.Unlock()
m.state.ShutDownCalled++
}
// ShutDownWithDrain implements [TypedInterface].
func (m *Mock[T]) ShutDownWithDrain() {
m.mutex.Lock()
defer m.mutex.Unlock()
m.state.ShutDownWithDrainCalled++
}
// ShuttingDown implements [TypedInterface].
func (m *Mock[T]) ShuttingDown() bool {
m.mutex.Lock()
defer m.mutex.Unlock()
return m.state.ShutDownCalled > 0 || m.state.ShutDownWithDrainCalled > 0
}
// AddAfter implements [TypedDelayingInterface.AddAfter]
func (m *Mock[T]) AddAfter(item T, duration time.Duration) {
if duration == 0 {
m.Add(item)
return
}
m.mutex.Lock()
defer m.mutex.Unlock()
for i := range m.state.Later {
if m.state.Later[i].Item == item {
// https://github.com/kubernetes/client-go/blob/270e5ab1714527c455865953da8ceba2810dbb50/util/workqueue/delaying_queue.go#L340-L349
// only shortens the delay for an existing item. It does not make it longer.
if m.state.Later[i].Duration > duration {
m.state.Later[i].Duration = duration
}
return
}
}
m.state.Later = append(m.state.Later, MockDelayedItem[T]{Item: item, Duration: duration})
}
// AddRateLimited implements [TypedRateLimitingInterface.AddRateLimited].
func (m *Mock[T]) AddRateLimited(item T) {
m.mutex.Lock()
defer m.mutex.Unlock()
if m.state.Failures == nil {
m.state.Failures = make(map[T]int)
}
m.state.Failures[item]++
}
// Forget implements [TypedRateLimitingInterface.Forget].
func (m *Mock[T]) Forget(item T) {
m.mutex.Lock()
defer m.mutex.Unlock()
if m.state.Failures == nil {
return
}
delete(m.state.Failures, item)
}
// NumRequeues implements [TypedRateLimitingInterface.NumRequeues].
func (m *Mock[T]) NumRequeues(item T) int {
m.mutex.Lock()
defer m.mutex.Unlock()
return m.state.Failures[item]
}

View File

@@ -393,7 +393,9 @@ func (d *draPlugin) PublishResources(ctx context.Context, resources Resources) e
driverResources := &resourceslice.DriverResources{ driverResources := &resourceslice.DriverResources{
Pools: map[string]resourceslice.Pool{ Pools: map[string]resourceslice.Pool{
d.nodeName: { d.nodeName: {
Slices: []resourceslice.Slice{{
Devices: resources.Devices, Devices: resources.Devices,
}},
}, },
}, },
} }
@@ -407,7 +409,13 @@ func (d *draPlugin) PublishResources(ctx context.Context, resources Resources) e
controllerLogger = klog.LoggerWithName(controllerLogger, "ResourceSlice controller") controllerLogger = klog.LoggerWithName(controllerLogger, "ResourceSlice controller")
controllerCtx = klog.NewContext(controllerCtx, controllerLogger) controllerCtx = klog.NewContext(controllerCtx, controllerLogger)
var err error var err error
if d.resourceSliceController, err = resourceslice.StartController(controllerCtx, d.kubeClient, d.driverName, owner, driverResources); err != nil { if d.resourceSliceController, err = resourceslice.StartController(controllerCtx,
resourceslice.Options{
DriverName: d.driverName,
KubeClient: d.kubeClient,
Owner: &owner,
Resources: driverResources,
}); err != nil {
return fmt.Errorf("start ResourceSlice controller: %w", err) return fmt.Errorf("start ResourceSlice controller: %w", err)
} }
return nil return nil

View File

@@ -20,19 +20,20 @@ import (
"context" "context"
"errors" "errors"
"fmt" "fmt"
"sort"
"sync" "sync"
"time" "time"
"github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp"
"k8s.io/api/core/v1" v1 "k8s.io/api/core/v1"
resourceapi "k8s.io/api/resource/v1alpha3" resourceapi "k8s.io/api/resource/v1alpha3"
apiequality "k8s.io/apimachinery/pkg/api/equality"
apierrors "k8s.io/apimachinery/pkg/api/errors" apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/fields"
"k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/types"
utilruntime "k8s.io/apimachinery/pkg/util/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/sets"
resourceinformers "k8s.io/client-go/informers/resource/v1alpha3" resourceinformers "k8s.io/client-go/informers/resource/v1alpha3"
"k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes"
"k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/cache"
@@ -57,8 +58,8 @@ const (
// controller as part of its kubelet plugin. // controller as part of its kubelet plugin.
type Controller struct { type Controller struct {
cancel func(cause error) cancel func(cause error)
driver string driverName string
owner Owner owner *Owner
kubeClient kubernetes.Interface kubeClient kubernetes.Interface
wg sync.WaitGroup wg sync.WaitGroup
// The queue is keyed with the pool name that needs work. // The queue is keyed with the pool name that needs work.
@@ -94,7 +95,28 @@ type Pool struct {
// by the controller. // by the controller.
Generation int64 Generation int64
// Device names must be unique inside the pool. // Slices is a list of all ResourceSlices that the driver
// wants to publish for this pool. The driver must ensure
// that each resulting slice is valid. See the API
// definition for details, in particular the limit on
// the number of devices.
//
// If slices are not valid, then the controller will
// log errors produced by the apiserver.
//
// Drivers should publish at least one slice for each
// pool that they normally manage, even if that slice
// is empty. "Empty pool" is different from "no pool"
// because it shows that the driver is up-and-running
// and simply doesn't have any devices.
Slices []Slice
}
// +k8s:deepcopy-gen=true
// Slice is turned into one ResourceSlice by the controller.
type Slice struct {
// Devices lists all devices which are part of the slice.
Devices []resourceapi.Device Devices []resourceapi.Device
} }
@@ -110,19 +132,9 @@ type Owner struct {
} }
// StartController constructs a new controller and starts it. // StartController constructs a new controller and starts it.
// If the owner is a v1.Node, then the NodeName field in the func StartController(ctx context.Context, options Options) (*Controller, error) {
// ResourceSlice objects is set and used to identify objects
// managed by the controller. The UID is not needed in that
// case, the controller will determine it automatically.
//
// If a kubeClient is provided, then it synchronizes ResourceSlices
// with the resource information provided by plugins. Without it,
// the controller is inactive. This can happen when kubelet is run stand-alone
// without an apiserver. In that case we can't and don't need to publish
// ResourceSlices.
func StartController(ctx context.Context, kubeClient kubernetes.Interface, driver string, owner Owner, resources *DriverResources) (*Controller, error) {
logger := klog.FromContext(ctx) logger := klog.FromContext(ctx)
c, err := newController(ctx, kubeClient, driver, owner, resources) c, err := newController(ctx, options)
if err != nil { if err != nil {
return nil, fmt.Errorf("create controller: %w", err) return nil, fmt.Errorf("create controller: %w", err)
} }
@@ -134,15 +146,38 @@ func StartController(ctx context.Context, kubeClient kubernetes.Interface, drive
defer logger.V(3).Info("Stopping") defer logger.V(3).Info("Stopping")
c.run(ctx) c.run(ctx)
}() }()
// Sync each pool once.
for poolName := range resources.Pools {
c.queue.Add(poolName)
}
return c, nil return c, nil
} }
// Options contains various optional settings for [StartController].
type Options struct {
// DriverName is the required name of the DRA driver.
DriverName string
// KubeClient is used to read Node objects (if necessary) and to access
// ResourceSlices. It must be specified.
KubeClient kubernetes.Interface
// If the owner is a v1.Node, then the NodeName field in the
// ResourceSlice objects is set and used to identify objects
// managed by the controller. The UID is not needed in that
// case, the controller will determine it automatically.
//
// The owner must be cluster-scoped. This is not always possible,
// therefore it is optional. A driver without a owner must take
// care that remaining slices get deleted manually as part of
// a driver uninstall because garbage collection won't work.
Owner *Owner
// This is the initial desired set of slices. As with
// [Controller.Update], the controller takes ownership of the resources
// instance. The content must not get modified by the caller.
Resources *DriverResources
// Queue can be used to override the default work queue implementation.
Queue workqueue.TypedRateLimitingInterface[string]
}
// Stop cancels all background activity and blocks until the controller has stopped. // Stop cancels all background activity and blocks until the controller has stopped.
func (c *Controller) Stop() { func (c *Controller) Stop() {
if c == nil { if c == nil {
@@ -155,7 +190,8 @@ func (c *Controller) Stop() {
// Update sets the new desired state of the resource information. // Update sets the new desired state of the resource information.
// //
// The controller takes over ownership, so these resources must // The controller takes over ownership, so these resources must
// not get modified after this method returns. // not get modified after this method returns. [DriverResources.DeepCopy]
// can be used by the caller to clone some existing instance.
func (c *Controller) Update(resources *DriverResources) { func (c *Controller) Update(resources *DriverResources) {
c.mutex.Lock() c.mutex.Lock()
defer c.mutex.Unlock() defer c.mutex.Unlock()
@@ -174,28 +210,43 @@ func (c *Controller) Update(resources *DriverResources) {
} }
// newController creates a new controller. // newController creates a new controller.
func newController(ctx context.Context, kubeClient kubernetes.Interface, driver string, owner Owner, resources *DriverResources) (*Controller, error) { func newController(ctx context.Context, options Options) (*Controller, error) {
if kubeClient == nil { if options.KubeClient == nil {
return nil, fmt.Errorf("kubeClient is nil") return nil, errors.New("KubeClient is nil")
}
if options.DriverName == "" {
return nil, errors.New("DRA driver name is empty")
}
if options.Resources == nil {
return nil, errors.New("DriverResources are nil")
} }
ctx, cancel := context.WithCancelCause(ctx) ctx, cancel := context.WithCancelCause(ctx)
c := &Controller{ c := &Controller{
cancel: cancel, cancel: cancel,
kubeClient: kubeClient, kubeClient: options.KubeClient,
driver: driver, driverName: options.DriverName,
owner: owner, owner: options.Owner.DeepCopy(),
queue: workqueue.NewTypedRateLimitingQueueWithConfig( queue: options.Queue,
resources: options.Resources,
}
if c.queue == nil {
c.queue = workqueue.NewTypedRateLimitingQueueWithConfig(
workqueue.DefaultTypedControllerRateLimiter[string](), workqueue.DefaultTypedControllerRateLimiter[string](),
workqueue.TypedRateLimitingQueueConfig[string]{Name: "node_resource_slices"}, workqueue.TypedRateLimitingQueueConfig[string]{Name: "node_resource_slices"},
), )
resources: resources,
} }
if err := c.initInformer(ctx); err != nil { if err := c.initInformer(ctx); err != nil {
return nil, err return nil, err
} }
// Sync each desired pool once.
for poolName := range options.Resources.Pools {
c.queue.Add(poolName)
}
return c, nil return c, nil
} }
@@ -205,10 +256,10 @@ func (c *Controller) initInformer(ctx context.Context) error {
// We always filter by driver name, by node name only for node-local resources. // We always filter by driver name, by node name only for node-local resources.
selector := fields.Set{ selector := fields.Set{
resourceapi.ResourceSliceSelectorDriver: c.driver, resourceapi.ResourceSliceSelectorDriver: c.driverName,
resourceapi.ResourceSliceSelectorNodeName: "", resourceapi.ResourceSliceSelectorNodeName: "",
} }
if c.owner.APIVersion == "v1" && c.owner.Kind == "Node" { if c.owner != nil && c.owner.APIVersion == "v1" && c.owner.Kind == "Node" {
selector[resourceapi.ResourceSliceSelectorNodeName] = c.owner.Name selector[resourceapi.ResourceSliceSelectorNodeName] = c.owner.Name
} }
informer := resourceinformers.NewFilteredResourceSliceInformer(c.kubeClient, resyncPeriod, cache.Indexers{ informer := resourceinformers.NewFilteredResourceSliceInformer(c.kubeClient, resyncPeriod, cache.Indexers{
@@ -348,7 +399,7 @@ func (c *Controller) syncPool(ctx context.Context, poolName string) error {
// The result gets cached and is expected to not change while // The result gets cached and is expected to not change while
// the controller runs. // the controller runs.
var nodeName string var nodeName string
if c.owner.APIVersion == "v1" && c.owner.Kind == "Node" { if c.owner != nil && c.owner.APIVersion == "v1" && c.owner.Kind == "Node" {
nodeName = c.owner.Name nodeName = c.owner.Name
if c.owner.UID == "" { if c.owner.UID == "" {
node, err := c.kubeClient.CoreV1().Nodes().Get(ctx, c.owner.Name, metav1.GetOptions{}) node, err := c.kubeClient.CoreV1().Nodes().Get(ctx, c.owner.Name, metav1.GetOptions{})
@@ -360,8 +411,7 @@ func (c *Controller) syncPool(ctx context.Context, poolName string) error {
} }
} }
// Slices that don't match any driver resource can either be updated (if there // Slices that don't match any driver slice need to be deleted.
// are new driver resources that need to be stored) or they need to be deleted.
obsoleteSlices := make([]*resourceapi.ResourceSlice, 0, len(slices)) obsoleteSlices := make([]*resourceapi.ResourceSlice, 0, len(slices))
// Determine highest generation. // Determine highest generation.
@@ -381,66 +431,162 @@ func (c *Controller) syncPool(ctx context.Context, poolName string) error {
currentSlices = append(currentSlices, slice) currentSlices = append(currentSlices, slice)
} }
} }
slices = currentSlices logger.V(5).Info("Existing slices", "obsolete", klog.KObjSlice(obsoleteSlices), "current", klog.KObjSlice(currentSlices))
// Sort by name to ensure that keeping only the first slice is deterministic.
sort.Slice(slices, func(i, j int) bool {
return slices[i].Name < slices[j].Name
})
if pool, ok := resources.Pools[poolName]; ok { if pool, ok := resources.Pools[poolName]; ok {
if pool.Generation > generation { // Match each existing slice against the desired slices.
generation = pool.Generation // Two slices match if they contain exactly the same
} // device IDs, in an arbitrary order. Such a matched
// slice gets updated with the desired content if
// Right now all devices get published in a single slice. // there is a difference.
// We simply pick the first one, if there is one, and copy
// it in preparation for updating it.
// //
// TODO: support splitting across slices, with unit tests. // This supports updating the definition of devices
if len(slices) > 0 { // in a slice. Adding or removing devices is done
obsoleteSlices = append(obsoleteSlices, slices[1:]...) // by deleting the old slice and creating a new one.
slices = []*resourceapi.ResourceSlice{slices[0].DeepCopy()} //
} else { // This is primarily a simplification of the code:
slices = []*resourceapi.ResourceSlice{ // to support adding or removing devices from
{ // existing slices, we would have to identify "most
ObjectMeta: metav1.ObjectMeta{ // similar" slices (= minimal editing distance).
GenerateName: c.owner.Name + "-" + c.driver + "-", //
}, // In currentSliceForDesiredSlice we keep track of
}, // which desired slice has a matched slice.
//
// At the end of the loop, each current slice is either
// a match or obsolete.
currentSliceForDesiredSlice := make(map[int]*resourceapi.ResourceSlice, len(pool.Slices))
for _, currentSlice := range currentSlices {
matched := false
for i := range pool.Slices {
if _, ok := currentSliceForDesiredSlice[i]; ok {
// Already has a match.
continue
}
if sameSlice(currentSlice, &pool.Slices[i]) {
currentSliceForDesiredSlice[i] = currentSlice
logger.V(5).Info("Matched existing slice", "slice", klog.KObj(currentSlice), "matchIndex", i)
matched = true
break
}
}
if !matched {
obsoleteSlices = append(obsoleteSlices, currentSlice)
logger.V(5).Info("Unmatched existing slice", "slice", klog.KObj(currentSlice))
} }
} }
slice := slices[0] // Desired metadata which must be set in each slice.
slice.OwnerReferences = []metav1.OwnerReference{{ resourceSliceCount := len(pool.Slices)
numMatchedSlices := len(currentSliceForDesiredSlice)
numNewSlices := resourceSliceCount - numMatchedSlices
desiredPool := resourceapi.ResourcePool{
Name: poolName,
Generation: generation, // May get updated later.
ResourceSliceCount: int64(resourceSliceCount),
}
desiredAllNodes := pool.NodeSelector == nil && nodeName == ""
// Now for each desired slice, figure out which of them are changed.
changedDesiredSlices := sets.New[int]()
for i, currentSlice := range currentSliceForDesiredSlice {
// Reordering entries is a difference and causes an update even if the
// entries are the same.
if !apiequality.Semantic.DeepEqual(&currentSlice.Spec.Pool, &desiredPool) ||
!apiequality.Semantic.DeepEqual(currentSlice.Spec.NodeSelector, pool.NodeSelector) ||
currentSlice.Spec.AllNodes != desiredAllNodes ||
!apiequality.Semantic.DeepEqual(currentSlice.Spec.Devices, pool.Slices[i].Devices) {
changedDesiredSlices.Insert(i)
logger.V(5).Info("Need to update slice", "slice", klog.KObj(currentSlice), "matchIndex", i)
}
}
logger.V(5).Info("Completed comparison",
"numObsolete", len(obsoleteSlices),
"numMatchedSlices", len(currentSliceForDesiredSlice),
"numChangedMatchedSlices", len(changedDesiredSlices),
"numNewSlices", numNewSlices,
)
bumpedGeneration := false
switch {
case pool.Generation > generation:
// Bump up the generation if the driver asked for it, or
// start with a non-zero generation.
generation = pool.Generation
bumpedGeneration = true
logger.V(5).Info("Bumped generation to driver-provided generation", "generation", generation)
case numNewSlices == 0 && len(changedDesiredSlices) <= 1:
logger.V(5).Info("Kept generation because at most one update API call is necessary", "generation", generation)
default:
generation++
bumpedGeneration = true
logger.V(5).Info("Bumped generation by one", "generation", generation)
}
desiredPool.Generation = generation
// Update existing slices.
for i, currentSlice := range currentSliceForDesiredSlice {
if !changedDesiredSlices.Has(i) && !bumpedGeneration {
continue
}
slice := currentSlice.DeepCopy()
slice.Spec.Pool = desiredPool
// No need to set the node name. If it was different, we wouldn't
// have listed the existing slice.
slice.Spec.NodeSelector = pool.NodeSelector
slice.Spec.AllNodes = desiredAllNodes
slice.Spec.Devices = pool.Slices[i].Devices
logger.V(5).Info("Updating existing resource slice", "slice", klog.KObj(slice))
if _, err := c.kubeClient.ResourceV1alpha3().ResourceSlices().Update(ctx, slice, metav1.UpdateOptions{}); err != nil {
return fmt.Errorf("update resource slice: %w", err)
}
}
// Create new slices.
for i := 0; i < len(pool.Slices); i++ {
if _, ok := currentSliceForDesiredSlice[i]; ok {
// Was handled above through an update.
continue
}
var ownerReferences []metav1.OwnerReference
if c.owner != nil {
ownerReferences = append(ownerReferences,
metav1.OwnerReference{
APIVersion: c.owner.APIVersion, APIVersion: c.owner.APIVersion,
Kind: c.owner.Kind, Kind: c.owner.Kind,
Name: c.owner.Name, Name: c.owner.Name,
UID: c.owner.UID, UID: c.owner.UID,
Controller: ptr.To(true), Controller: ptr.To(true),
}} },
slice.Spec.Driver = c.driver )
slice.Spec.Pool.Name = poolName }
slice.Spec.Pool.Generation = generation generateName := c.driverName + "-"
slice.Spec.Pool.ResourceSliceCount = 1 if c.owner != nil {
slice.Spec.NodeName = nodeName generateName = c.owner.Name + "-" + generateName
slice.Spec.NodeSelector = pool.NodeSelector }
slice.Spec.AllNodes = pool.NodeSelector == nil && nodeName == "" slice := &resourceapi.ResourceSlice{
slice.Spec.Devices = pool.Devices ObjectMeta: metav1.ObjectMeta{
OwnerReferences: ownerReferences,
if loggerV := logger.V(6); loggerV.Enabled() { GenerateName: generateName,
// Dump entire resource information. },
loggerV.Info("Syncing resource slices", "obsoleteSlices", klog.KObjSlice(obsoleteSlices), "slices", klog.KObjSlice(slices), "pool", pool) Spec: resourceapi.ResourceSliceSpec{
} else { Driver: c.driverName,
logger.V(5).Info("Syncing resource slices", "obsoleteSlices", klog.KObjSlice(obsoleteSlices), "slices", klog.KObjSlice(slices), "numDevices", len(pool.Devices)) Pool: desiredPool,
NodeName: nodeName,
NodeSelector: pool.NodeSelector,
AllNodes: desiredAllNodes,
Devices: pool.Slices[i].Devices,
},
}
logger.V(5).Info("Creating new resource slice")
if _, err := c.kubeClient.ResourceV1alpha3().ResourceSlices().Create(ctx, slice, metav1.CreateOptions{}); err != nil {
return fmt.Errorf("create resource slice: %w", err)
}
} }
} else if len(slices) > 0 { } else if len(slices) > 0 {
// All are obsolete, pool does not exist anymore. // All are obsolete, pool does not exist anymore.
obsoleteSlices = slices
logger.V(5).Info("Removing resource slices after pool removal", "obsoleteSlices", klog.KObjSlice(obsoleteSlices), "slices", klog.KObjSlice(slices), "numDevices", len(pool.Devices)) logger.V(5).Info("Removing resource slices after pool removal")
obsoleteSlices = append(obsoleteSlices, slices...)
// No need to create or update the slices.
slices = nil
} }
// Remove stale slices. // Remove stale slices.
@@ -451,22 +597,24 @@ func (c *Controller) syncPool(ctx context.Context, poolName string) error {
} }
} }
// Create or update slices.
for _, slice := range slices {
if slice.UID == "" {
logger.V(5).Info("Creating new resource slice", "slice", klog.KObj(slice))
if _, err := c.kubeClient.ResourceV1alpha3().ResourceSlices().Create(ctx, slice, metav1.CreateOptions{}); err != nil {
return fmt.Errorf("create resource slice: %w", err)
}
continue
}
// TODO: switch to SSA once unit testing supports it.
logger.V(5).Info("Updating existing resource slice", "slice", klog.KObj(slice))
if _, err := c.kubeClient.ResourceV1alpha3().ResourceSlices().Update(ctx, slice, metav1.UpdateOptions{}); err != nil {
return fmt.Errorf("update resource slice: %w", err)
}
}
return nil return nil
} }
func sameSlice(existingSlice *resourceapi.ResourceSlice, desiredSlice *Slice) bool {
if len(existingSlice.Spec.Devices) != len(desiredSlice.Devices) {
return false
}
existingDevices := sets.New[string]()
for _, device := range existingSlice.Spec.Devices {
existingDevices.Insert(device.Name)
}
for _, device := range desiredSlice.Devices {
if !existingDevices.Has(device.Name) {
return false
}
}
// Same number of devices, names all present -> equal.
return true
}

View File

@@ -17,6 +17,10 @@ limitations under the License.
package resourceslice package resourceslice
import ( import (
"fmt"
"sort"
"strconv"
"sync"
"testing" "testing"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
@@ -28,217 +32,709 @@ import (
"k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/kubernetes/fake" "k8s.io/client-go/kubernetes/fake"
k8stesting "k8s.io/client-go/testing"
"k8s.io/dynamic-resource-allocation/internal/workqueue"
"k8s.io/klog/v2"
"k8s.io/klog/v2/ktesting" "k8s.io/klog/v2/ktesting"
"k8s.io/utils/ptr" "k8s.io/utils/ptr"
) )
func init() {
klog.InitFlags(nil)
}
// TestControllerSyncPool verifies that syncPool produces the right ResourceSlices.
// Update vs. Create API calls are checked by bumping the ResourceVersion in
// updates.
func TestControllerSyncPool(t *testing.T) { func TestControllerSyncPool(t *testing.T) {
var ( var (
node = "node" ownerName = "owner"
nodeUID = types.UID("node-uid") nodeUID = types.UID("node-uid")
driveName = "driver" driverName = "driver"
poolName = "pool" poolName = "pool"
poolName1 = "pool-1"
deviceName = "device" deviceName = "device"
deviceName1 = "device-1"
deviceName2 = "device-2"
deviceName3 = "device-3"
deviceName4 = "device-4"
resourceSlice1 = "resource-slice-1" resourceSlice1 = "resource-slice-1"
resourceSlice2 = "resource-slice-2" resourceSlice2 = "resource-slice-2"
resourceSlice3 = "resource-slice-3" resourceSlice3 = "resource-slice-3"
generateName = ownerName + "-" + driverName + "-"
generatedName1 = generateName + "0"
basicDevice = &resourceapi.BasicDevice{
Attributes: map[resourceapi.QualifiedName]resourceapi.DeviceAttribute{
"new-attribute": {StringValue: ptr.To("value")},
},
}
nodeSelector = &v1.NodeSelector{
NodeSelectorTerms: []v1.NodeSelectorTerm{{
MatchFields: []v1.NodeSelectorRequirement{{
Key: "name",
Values: []string{"node-a"},
}},
}},
}
otherNodeSelector = &v1.NodeSelector{
NodeSelectorTerms: []v1.NodeSelectorTerm{{
MatchFields: []v1.NodeSelectorRequirement{{
Key: "name",
Values: []string{"node-b"},
}},
}},
}
) )
testCases := map[string]struct { testCases := map[string]struct {
// nodeUID is empty if not a node-local.
nodeUID types.UID nodeUID types.UID
// noOwner completely disables setting an owner.
noOwner bool
// initialObjects is a list of initial resource slices to be used in the test.
initialObjects []runtime.Object initialObjects []runtime.Object
poolName string initialOtherObjects []runtime.Object
inputDriverResources *DriverResources inputDriverResources *DriverResources
expectedResourceSlices []resourceapi.ResourceSlice expectedResourceSlices []resourceapi.ResourceSlice
}{ }{
"single-resourceslice-existing": { "create-slice": {
nodeUID: nodeUID,
initialObjects: []runtime.Object{
newResourceSlice(resourceSlice1, node, driveName, poolName, 1),
},
poolName: poolName,
inputDriverResources: &DriverResources{
Pools: map[string]Pool{
poolName: {
Devices: []resourceapi.Device{
{
Name: deviceName,
},
},
},
},
},
expectedResourceSlices: []resourceapi.ResourceSlice{
*newExpectResourceSlice(resourceSlice1, node, string(nodeUID), driveName, poolName, deviceName, false, 1),
},
},
"single-resourceslice-existing-with-nodeUID-empty": {
nodeUID: "",
initialObjects: []runtime.Object{
newNode(node, nodeUID),
newResourceSlice(resourceSlice1, node, driveName, poolName, 1),
},
poolName: poolName,
inputDriverResources: &DriverResources{
Pools: map[string]Pool{
poolName: {
Devices: []resourceapi.Device{
{
Name: deviceName,
},
},
},
},
},
expectedResourceSlices: []resourceapi.ResourceSlice{
*newExpectResourceSlice(resourceSlice1, node, string(nodeUID), driveName, poolName, deviceName, false, 1),
},
},
"multiple-resourceslices-existing": {
nodeUID: nodeUID,
initialObjects: []runtime.Object{
newResourceSlice(resourceSlice1, node, driveName, poolName, 1),
newResourceSlice(resourceSlice2, node, driveName, poolName, 1),
newResourceSlice(resourceSlice3, node, driveName, poolName, 1),
},
poolName: poolName,
inputDriverResources: &DriverResources{
Pools: map[string]Pool{
poolName: {
Devices: []resourceapi.Device{
{
Name: deviceName,
},
},
},
},
},
expectedResourceSlices: []resourceapi.ResourceSlice{
*newExpectResourceSlice(resourceSlice1, node, string(nodeUID), driveName, poolName, deviceName, false, 1),
},
},
"no-resourceslices-existing": {
nodeUID: nodeUID, nodeUID: nodeUID,
initialObjects: []runtime.Object{}, initialObjects: []runtime.Object{},
poolName: poolName,
inputDriverResources: &DriverResources{ inputDriverResources: &DriverResources{
Pools: map[string]Pool{ Pools: map[string]Pool{
poolName: { poolName: {
Devices: []resourceapi.Device{ Slices: []Slice{{Devices: []resourceapi.Device{}}},
{
Name: deviceName,
},
},
}, },
}, },
}, },
expectedResourceSlices: []resourceapi.ResourceSlice{ expectedResourceSlices: []resourceapi.ResourceSlice{
*newExpectResourceSlice(node+"-"+driveName+"-", node, string(nodeUID), driveName, poolName, deviceName, true, 0), *MakeResourceSlice().Name(generatedName1).GenerateName(generateName).
NodeOwnerReferences(ownerName, string(nodeUID)).NodeName(ownerName).
Driver(driverName).Devices([]resourceapi.Device{}).
Pool(resourceapi.ResourcePool{Name: poolName, Generation: 1, ResourceSliceCount: 1}).Obj(),
}, },
}, },
"single-resourceslice-existing-with-different-resource-pool-generation": { "keep-slice-unchanged": {
nodeUID: nodeUID, nodeUID: nodeUID,
initialObjects: []runtime.Object{ initialObjects: []runtime.Object{
newResourceSlice(resourceSlice1, node, driveName, poolName, 2), MakeResourceSlice().Name(generatedName1).GenerateName(generateName).
newResourceSlice(resourceSlice2, node, driveName, poolName, 1), NodeOwnerReferences(ownerName, string(nodeUID)).NodeName(ownerName).
newResourceSlice(resourceSlice3, node, driveName, poolName, 1), Driver(driverName).Devices([]resourceapi.Device{{Name: deviceName}}).
Pool(resourceapi.ResourcePool{Name: poolName, Generation: 1, ResourceSliceCount: 1}).Obj(),
}, },
poolName: poolName,
inputDriverResources: &DriverResources{ inputDriverResources: &DriverResources{
Pools: map[string]Pool{ Pools: map[string]Pool{
poolName: { poolName: {
Generation: 3,
Devices: []resourceapi.Device{
{
Name: deviceName,
},
},
},
},
},
expectedResourceSlices: []resourceapi.ResourceSlice{
*newExpectResourceSlice(resourceSlice1, node, string(nodeUID), driveName, poolName, deviceName, false, 3),
},
},
"single-resourceslice-existing-with-mismatching-resource-pool-name": {
nodeUID: nodeUID,
initialObjects: []runtime.Object{
newResourceSlice(resourceSlice1, node, driveName, poolName, 1),
},
poolName: poolName,
inputDriverResources: &DriverResources{
Pools: map[string]Pool{
poolName1: {
Generation: 1, Generation: 1,
Slices: []Slice{{Devices: []resourceapi.Device{{Name: deviceName}}}},
},
},
},
expectedResourceSlices: []resourceapi.ResourceSlice{
*MakeResourceSlice().Name(generatedName1).GenerateName(generateName).
NodeOwnerReferences(ownerName, string(nodeUID)).NodeName(ownerName).
Driver(driverName).Devices([]resourceapi.Device{{Name: deviceName}}).
Pool(resourceapi.ResourcePool{Name: poolName, Generation: 1, ResourceSliceCount: 1}).Obj(),
},
},
"remove-pool": {
nodeUID: nodeUID,
initialObjects: []runtime.Object{
MakeResourceSlice().Name(resourceSlice1).UID(resourceSlice1).
NodeOwnerReferences(ownerName, string(nodeUID)).NodeName(ownerName).
Driver(driverName).Devices([]resourceapi.Device{}).
Pool(resourceapi.ResourcePool{Name: poolName, Generation: 1, ResourceSliceCount: 1}).Obj(),
},
inputDriverResources: &DriverResources{},
expectedResourceSlices: nil,
},
"delete-and-add-slice": {
nodeUID: nodeUID,
initialObjects: []runtime.Object{
// no devices
MakeResourceSlice().Name(resourceSlice1).UID(resourceSlice1).
NodeOwnerReferences(ownerName, string(nodeUID)).NodeName(ownerName).
Driver(driverName).Devices([]resourceapi.Device{}).
Pool(resourceapi.ResourcePool{Name: poolName, Generation: 1, ResourceSliceCount: 1}).Obj(),
},
inputDriverResources: &DriverResources{
Pools: map[string]Pool{
poolName: {
Slices: []Slice{{Devices: []resourceapi.Device{{Name: deviceName}}}}},
},
},
expectedResourceSlices: []resourceapi.ResourceSlice{
*MakeResourceSlice().Name(generatedName1).GenerateName(generateName).
NodeOwnerReferences(ownerName, string(nodeUID)).NodeName(ownerName).
Driver(driverName).Devices([]resourceapi.Device{{Name: deviceName}}).
Pool(resourceapi.ResourcePool{Name: poolName, Generation: 2, ResourceSliceCount: 1}).Obj(),
},
},
"delete-redundant-slice": {
nodeUID: nodeUID,
initialObjects: []runtime.Object{
MakeResourceSlice().Name(resourceSlice1).UID(resourceSlice1).
NodeOwnerReferences(ownerName, string(nodeUID)).NodeName(ownerName).
Driver(driverName).Devices([]resourceapi.Device{{Name: deviceName}}).
Pool(resourceapi.ResourcePool{Name: poolName, Generation: 1, ResourceSliceCount: 1}).Obj(),
MakeResourceSlice().Name(resourceSlice2).UID(resourceSlice2).
NodeOwnerReferences(ownerName, string(nodeUID)).NodeName(ownerName).
Driver(driverName).Devices([]resourceapi.Device{{Name: deviceName}}).
Pool(resourceapi.ResourcePool{Name: poolName, Generation: 1, ResourceSliceCount: 1}).Obj(),
},
inputDriverResources: &DriverResources{
Pools: map[string]Pool{
poolName: {
Slices: []Slice{{Devices: []resourceapi.Device{{Name: deviceName}}}}},
},
},
expectedResourceSlices: []resourceapi.ResourceSlice{
*MakeResourceSlice().Name(resourceSlice1).UID(resourceSlice1).
NodeOwnerReferences(ownerName, string(nodeUID)).NodeName(ownerName).
Driver(driverName).Devices([]resourceapi.Device{{Name: deviceName}}).
Pool(resourceapi.ResourcePool{Name: poolName, Generation: 1, ResourceSliceCount: 1}).Obj(),
},
},
"update-slice": {
nodeUID: nodeUID,
initialObjects: []runtime.Object{
MakeResourceSlice().Name(resourceSlice1).UID(resourceSlice1).
NodeOwnerReferences(ownerName, string(nodeUID)).NodeName(ownerName).
Driver(driverName).Devices([]resourceapi.Device{{Name: deviceName}}).
Pool(resourceapi.ResourcePool{Name: poolName, Generation: 1, ResourceSliceCount: 1}).Obj(),
},
inputDriverResources: &DriverResources{
Pools: map[string]Pool{
poolName: {
Slices: []Slice{{
Devices: []resourceapi.Device{{
Name: deviceName,
Basic: basicDevice,
}},
}},
},
},
},
expectedResourceSlices: []resourceapi.ResourceSlice{
*MakeResourceSlice().Name(resourceSlice1).UID(resourceSlice1).ResourceVersion("1").
NodeOwnerReferences(ownerName, string(nodeUID)).NodeName(ownerName).
Driver(driverName).Devices([]resourceapi.Device{{
Name: deviceName,
Basic: basicDevice,
}}).
Pool(resourceapi.ResourcePool{Name: poolName, Generation: 1, ResourceSliceCount: 1}).Obj(),
},
},
"update-slice-many-devices": {
nodeUID: nodeUID,
initialObjects: []runtime.Object{
MakeResourceSlice().Name(resourceSlice1).UID(resourceSlice1).
NodeOwnerReferences(ownerName, string(nodeUID)).NodeName(ownerName).
Driver(driverName).Devices([]resourceapi.Device{{Name: deviceName1}, {Name: deviceName2}}).
Pool(resourceapi.ResourcePool{Name: poolName, Generation: 1, ResourceSliceCount: 1}).Obj(),
},
inputDriverResources: &DriverResources{
Pools: map[string]Pool{
poolName: {
Slices: []Slice{{
Devices: []resourceapi.Device{
{Name: deviceName1},
{
Name: deviceName2,
Basic: basicDevice,
},
},
}},
},
},
},
expectedResourceSlices: []resourceapi.ResourceSlice{
*MakeResourceSlice().Name(resourceSlice1).UID(resourceSlice1).ResourceVersion("1").
NodeOwnerReferences(ownerName, string(nodeUID)).NodeName(ownerName).
Driver(driverName).Devices([]resourceapi.Device{
{Name: deviceName1},
{
Name: deviceName2,
Basic: basicDevice,
}}).
Pool(resourceapi.ResourcePool{Name: poolName, Generation: 1, ResourceSliceCount: 1}).Obj(),
},
},
"multiple-resourceslices-existing-no-changes": {
nodeUID: nodeUID,
initialObjects: []runtime.Object{
MakeResourceSlice().Name(resourceSlice1).UID(resourceSlice1).
NodeOwnerReferences(ownerName, string(nodeUID)).NodeName(ownerName).
Driver(driverName).Devices([]resourceapi.Device{{Name: deviceName1}}).
Pool(resourceapi.ResourcePool{Name: poolName, Generation: 1, ResourceSliceCount: 3}).Obj(),
MakeResourceSlice().Name(resourceSlice2).UID(resourceSlice2).
NodeOwnerReferences(ownerName, string(nodeUID)).NodeName(ownerName).
Driver(driverName).Devices([]resourceapi.Device{{Name: deviceName2}}).
Pool(resourceapi.ResourcePool{Name: poolName, Generation: 1, ResourceSliceCount: 3}).Obj(),
MakeResourceSlice().Name(resourceSlice3).UID(resourceSlice3).
NodeOwnerReferences(ownerName, string(nodeUID)).NodeName(ownerName).
Driver(driverName).Devices([]resourceapi.Device{{Name: deviceName3}}).
Pool(resourceapi.ResourcePool{Name: poolName, Generation: 1, ResourceSliceCount: 3}).Obj(),
},
inputDriverResources: &DriverResources{
Pools: map[string]Pool{
poolName: {
Slices: []Slice{
{Devices: []resourceapi.Device{{Name: deviceName1}}},
{Devices: []resourceapi.Device{{Name: deviceName2}}},
{Devices: []resourceapi.Device{{Name: deviceName3}}},
},
},
},
},
expectedResourceSlices: []resourceapi.ResourceSlice{
*MakeResourceSlice().Name(resourceSlice1).UID(resourceSlice1).
NodeOwnerReferences(ownerName, string(nodeUID)).NodeName(ownerName).
Driver(driverName).Devices([]resourceapi.Device{{Name: deviceName1}}).
Pool(resourceapi.ResourcePool{Name: poolName, Generation: 1, ResourceSliceCount: 3}).Obj(),
*MakeResourceSlice().Name(resourceSlice2).UID(resourceSlice2).
NodeOwnerReferences(ownerName, string(nodeUID)).NodeName(ownerName).
Driver(driverName).Devices([]resourceapi.Device{{Name: deviceName2}}).
Pool(resourceapi.ResourcePool{Name: poolName, Generation: 1, ResourceSliceCount: 3}).Obj(),
*MakeResourceSlice().Name(resourceSlice3).UID(resourceSlice3).
NodeOwnerReferences(ownerName, string(nodeUID)).NodeName(ownerName).
Driver(driverName).Devices([]resourceapi.Device{{Name: deviceName3}}).
Pool(resourceapi.ResourcePool{Name: poolName, Generation: 1, ResourceSliceCount: 3}).Obj(),
},
},
"multiple-resourceslices-existing-with-different-resource-pool-generation": {
nodeUID: nodeUID,
initialObjects: []runtime.Object{
// no devices
MakeResourceSlice().Name(resourceSlice1).UID(resourceSlice1).
NodeOwnerReferences(ownerName, string(nodeUID)).NodeName(ownerName).
Driver(driverName).Devices([]resourceapi.Device{}).
Pool(resourceapi.ResourcePool{Name: poolName, Generation: 1, ResourceSliceCount: 1}).Obj(),
// matching device
MakeResourceSlice().Name(resourceSlice2).UID(resourceSlice2).
NodeOwnerReferences(ownerName, string(nodeUID)).NodeName(ownerName).
Driver(driverName).Devices([]resourceapi.Device{{Name: deviceName}}).
Pool(resourceapi.ResourcePool{Name: poolName, Generation: 2, ResourceSliceCount: 1}).Obj(),
// no devices
MakeResourceSlice().Name(resourceSlice3).UID(resourceSlice3).
NodeOwnerReferences(ownerName, string(nodeUID)).NodeName(ownerName).
Driver(driverName).Devices([]resourceapi.Device{}).
Pool(resourceapi.ResourcePool{Name: poolName, Generation: 1, ResourceSliceCount: 1}).Obj(),
},
inputDriverResources: &DriverResources{
Pools: map[string]Pool{
poolName: {
Slices: []Slice{{
Devices: []resourceapi.Device{ Devices: []resourceapi.Device{
{ {
Name: deviceName, Name: deviceName,
}, },
}, },
}},
}, },
}, },
}, },
expectedResourceSlices: nil, expectedResourceSlices: []resourceapi.ResourceSlice{
*MakeResourceSlice().Name(resourceSlice2).UID(resourceSlice2).
NodeOwnerReferences(ownerName, string(nodeUID)).NodeName(ownerName).
Driver(driverName).Devices([]resourceapi.Device{{Name: deviceName}}).
Pool(resourceapi.ResourcePool{Name: poolName, Generation: 2, ResourceSliceCount: 1}).Obj(),
},
},
"multiple-resourceslices-existing-one-changed": {
nodeUID: nodeUID,
initialObjects: []runtime.Object{
MakeResourceSlice().Name(resourceSlice1).UID(resourceSlice1).
NodeOwnerReferences(ownerName, string(nodeUID)).NodeName(ownerName).
Driver(driverName).Devices([]resourceapi.Device{{Name: deviceName1}}).
Pool(resourceapi.ResourcePool{Name: poolName, Generation: 1, ResourceSliceCount: 3}).Obj(),
MakeResourceSlice().Name(resourceSlice2).UID(resourceSlice2).
NodeOwnerReferences(ownerName, string(nodeUID)).NodeName(ownerName).
Driver(driverName).Devices([]resourceapi.Device{{Name: deviceName2}}).
Pool(resourceapi.ResourcePool{Name: poolName, Generation: 1, ResourceSliceCount: 3}).Obj(),
MakeResourceSlice().Name(resourceSlice3).UID(resourceSlice3).
NodeOwnerReferences(ownerName, string(nodeUID)).NodeName(ownerName).
Driver(driverName).Devices([]resourceapi.Device{{Name: deviceName3}}).
Pool(resourceapi.ResourcePool{Name: poolName, Generation: 1, ResourceSliceCount: 3}).Obj(),
},
inputDriverResources: &DriverResources{
Pools: map[string]Pool{
poolName: {
Slices: []Slice{
{Devices: []resourceapi.Device{{Name: deviceName1}}},
{Devices: []resourceapi.Device{{Name: deviceName2, Basic: basicDevice}}},
{Devices: []resourceapi.Device{{Name: deviceName3}}},
},
},
},
},
// Generation not bumped, only one update.
expectedResourceSlices: []resourceapi.ResourceSlice{
*MakeResourceSlice().Name(resourceSlice1).UID(resourceSlice1).
NodeOwnerReferences(ownerName, string(nodeUID)).NodeName(ownerName).
Driver(driverName).Devices([]resourceapi.Device{{Name: deviceName1}}).
Pool(resourceapi.ResourcePool{Name: poolName, Generation: 1, ResourceSliceCount: 3}).Obj(),
*MakeResourceSlice().Name(resourceSlice2).UID(resourceSlice2).ResourceVersion("1").
NodeOwnerReferences(ownerName, string(nodeUID)).NodeName(ownerName).
Driver(driverName).Devices([]resourceapi.Device{{Name: deviceName2, Basic: basicDevice}}).
Pool(resourceapi.ResourcePool{Name: poolName, Generation: 1, ResourceSliceCount: 3}).Obj(),
*MakeResourceSlice().Name(resourceSlice3).UID(resourceSlice3).
NodeOwnerReferences(ownerName, string(nodeUID)).NodeName(ownerName).
Driver(driverName).Devices([]resourceapi.Device{{Name: deviceName3}}).
Pool(resourceapi.ResourcePool{Name: poolName, Generation: 1, ResourceSliceCount: 3}).Obj(),
},
},
"multiple-resourceslices-existing-two-changed": {
nodeUID: nodeUID,
initialObjects: []runtime.Object{
MakeResourceSlice().Name(resourceSlice1).UID(resourceSlice1).
NodeOwnerReferences(ownerName, string(nodeUID)).NodeName(ownerName).
Driver(driverName).Devices([]resourceapi.Device{{Name: deviceName1}}).
Pool(resourceapi.ResourcePool{Name: poolName, Generation: 1, ResourceSliceCount: 3}).Obj(),
MakeResourceSlice().Name(resourceSlice2).UID(resourceSlice2).
NodeOwnerReferences(ownerName, string(nodeUID)).NodeName(ownerName).
Driver(driverName).Devices([]resourceapi.Device{{Name: deviceName2}}).
Pool(resourceapi.ResourcePool{Name: poolName, Generation: 1, ResourceSliceCount: 3}).Obj(),
MakeResourceSlice().Name(resourceSlice3).UID(resourceSlice3).
NodeOwnerReferences(ownerName, string(nodeUID)).NodeName(ownerName).
Driver(driverName).Devices([]resourceapi.Device{{Name: deviceName3}}).
Pool(resourceapi.ResourcePool{Name: poolName, Generation: 1, ResourceSliceCount: 3}).Obj(),
},
inputDriverResources: &DriverResources{
Pools: map[string]Pool{
poolName: {
Slices: []Slice{
{Devices: []resourceapi.Device{{Name: deviceName1}}},
{Devices: []resourceapi.Device{{Name: deviceName2, Basic: basicDevice}}},
{Devices: []resourceapi.Device{{Name: deviceName3, Basic: basicDevice}}},
},
},
},
},
// Generation bumped, all updated.
expectedResourceSlices: []resourceapi.ResourceSlice{
*MakeResourceSlice().Name(resourceSlice1).UID(resourceSlice1).
NodeOwnerReferences(ownerName, string(nodeUID)).NodeName(ownerName).ResourceVersion("1").
Driver(driverName).Devices([]resourceapi.Device{{Name: deviceName1}}).
Pool(resourceapi.ResourcePool{Name: poolName, Generation: 2, ResourceSliceCount: 3}).Obj(),
*MakeResourceSlice().Name(resourceSlice2).UID(resourceSlice2).ResourceVersion("1").
NodeOwnerReferences(ownerName, string(nodeUID)).NodeName(ownerName).
Driver(driverName).Devices([]resourceapi.Device{{Name: deviceName2, Basic: basicDevice}}).
Pool(resourceapi.ResourcePool{Name: poolName, Generation: 2, ResourceSliceCount: 3}).Obj(),
*MakeResourceSlice().Name(resourceSlice3).UID(resourceSlice3).
NodeOwnerReferences(ownerName, string(nodeUID)).NodeName(ownerName).ResourceVersion("1").
Driver(driverName).Devices([]resourceapi.Device{{Name: deviceName3, Basic: basicDevice}}).
Pool(resourceapi.ResourcePool{Name: poolName, Generation: 2, ResourceSliceCount: 3}).Obj(),
},
},
"multiple-resourceslices-existing-one-removed": {
nodeUID: nodeUID,
initialObjects: []runtime.Object{
MakeResourceSlice().Name(resourceSlice1).UID(resourceSlice1).
NodeOwnerReferences(ownerName, string(nodeUID)).NodeName(ownerName).
Driver(driverName).Devices([]resourceapi.Device{{Name: deviceName1}}).
Pool(resourceapi.ResourcePool{Name: poolName, Generation: 1, ResourceSliceCount: 3}).Obj(),
MakeResourceSlice().Name(resourceSlice2).UID(resourceSlice2).
NodeOwnerReferences(ownerName, string(nodeUID)).NodeName(ownerName).
Driver(driverName).Devices([]resourceapi.Device{{Name: deviceName2}}).
Pool(resourceapi.ResourcePool{Name: poolName, Generation: 1, ResourceSliceCount: 3}).Obj(),
MakeResourceSlice().Name(resourceSlice3).UID(resourceSlice3).
NodeOwnerReferences(ownerName, string(nodeUID)).NodeName(ownerName).
Driver(driverName).Devices([]resourceapi.Device{{Name: deviceName3}}).
Pool(resourceapi.ResourcePool{Name: poolName, Generation: 1, ResourceSliceCount: 3}).Obj(),
},
inputDriverResources: &DriverResources{
Pools: map[string]Pool{
poolName: {
Slices: []Slice{
{Devices: []resourceapi.Device{{Name: deviceName1}}},
{Devices: []resourceapi.Device{{Name: deviceName2}}},
},
},
},
},
// Generation bumped, two updated, one removed.
expectedResourceSlices: []resourceapi.ResourceSlice{
*MakeResourceSlice().Name(resourceSlice1).UID(resourceSlice1).
NodeOwnerReferences(ownerName, string(nodeUID)).NodeName(ownerName).ResourceVersion("1").
Driver(driverName).Devices([]resourceapi.Device{{Name: deviceName1}}).
Pool(resourceapi.ResourcePool{Name: poolName, Generation: 2, ResourceSliceCount: 2}).Obj(),
*MakeResourceSlice().Name(resourceSlice2).UID(resourceSlice2).ResourceVersion("1").
NodeOwnerReferences(ownerName, string(nodeUID)).NodeName(ownerName).
Driver(driverName).Devices([]resourceapi.Device{{Name: deviceName2}}).
Pool(resourceapi.ResourcePool{Name: poolName, Generation: 2, ResourceSliceCount: 2}).Obj(),
},
},
"multiple-resourceslices-existing-one-added": {
nodeUID: nodeUID,
initialObjects: []runtime.Object{
MakeResourceSlice().Name(resourceSlice1).UID(resourceSlice1).
NodeOwnerReferences(ownerName, string(nodeUID)).NodeName(ownerName).
Driver(driverName).Devices([]resourceapi.Device{{Name: deviceName1}}).
Pool(resourceapi.ResourcePool{Name: poolName, Generation: 1, ResourceSliceCount: 3}).Obj(),
MakeResourceSlice().Name(resourceSlice2).UID(resourceSlice2).
NodeOwnerReferences(ownerName, string(nodeUID)).NodeName(ownerName).
Driver(driverName).Devices([]resourceapi.Device{{Name: deviceName2}}).
Pool(resourceapi.ResourcePool{Name: poolName, Generation: 1, ResourceSliceCount: 3}).Obj(),
MakeResourceSlice().Name(resourceSlice3).UID(resourceSlice3).
NodeOwnerReferences(ownerName, string(nodeUID)).NodeName(ownerName).
Driver(driverName).Devices([]resourceapi.Device{{Name: deviceName3}}).
Pool(resourceapi.ResourcePool{Name: poolName, Generation: 1, ResourceSliceCount: 3}).Obj(),
},
inputDriverResources: &DriverResources{
Pools: map[string]Pool{
poolName: {
Slices: []Slice{
{Devices: []resourceapi.Device{{Name: deviceName1}}},
{Devices: []resourceapi.Device{{Name: deviceName2}}},
{Devices: []resourceapi.Device{{Name: deviceName3}}},
{Devices: []resourceapi.Device{{Name: deviceName4}}},
},
},
},
},
// Three updated, one generated.
expectedResourceSlices: []resourceapi.ResourceSlice{
*MakeResourceSlice().Name(resourceSlice1).UID(resourceSlice1).ResourceVersion("1").
NodeOwnerReferences(ownerName, string(nodeUID)).NodeName(ownerName).
Driver(driverName).Devices([]resourceapi.Device{{Name: deviceName1}}).
Pool(resourceapi.ResourcePool{Name: poolName, Generation: 2, ResourceSliceCount: 4}).Obj(),
*MakeResourceSlice().Name(resourceSlice2).UID(resourceSlice2).ResourceVersion("1").
NodeOwnerReferences(ownerName, string(nodeUID)).NodeName(ownerName).
Driver(driverName).Devices([]resourceapi.Device{{Name: deviceName2}}).
Pool(resourceapi.ResourcePool{Name: poolName, Generation: 2, ResourceSliceCount: 4}).Obj(),
*MakeResourceSlice().Name(resourceSlice3).UID(resourceSlice3).ResourceVersion("1").
NodeOwnerReferences(ownerName, string(nodeUID)).NodeName(ownerName).
Driver(driverName).Devices([]resourceapi.Device{{Name: deviceName3}}).
Pool(resourceapi.ResourcePool{Name: poolName, Generation: 2, ResourceSliceCount: 4}).Obj(),
*MakeResourceSlice().GenerateName(generateName).Name(generatedName1).
NodeOwnerReferences(ownerName, string(nodeUID)).NodeName(ownerName).
Driver(driverName).Devices([]resourceapi.Device{{Name: deviceName4}}).
Pool(resourceapi.ResourcePool{Name: poolName, Generation: 2, ResourceSliceCount: 4}).Obj(),
},
},
"add-one-network-device-all-nodes": {
initialObjects: []runtime.Object{},
noOwner: true,
inputDriverResources: &DriverResources{
Pools: map[string]Pool{
poolName: {
Slices: []Slice{{Devices: []resourceapi.Device{{Name: deviceName}}}},
},
},
},
expectedResourceSlices: []resourceapi.ResourceSlice{
*MakeResourceSlice().Name(driverName + "-0").GenerateName(driverName + "-").
AllNodes(true).
Driver(driverName).Devices([]resourceapi.Device{{Name: deviceName}}).
Pool(resourceapi.ResourcePool{Name: poolName, Generation: 1, ResourceSliceCount: 1}).Obj(),
},
},
"add-one-network-device-some-nodes": {
initialObjects: []runtime.Object{},
inputDriverResources: &DriverResources{
Pools: map[string]Pool{
poolName: {
NodeSelector: nodeSelector,
Slices: []Slice{{Devices: []resourceapi.Device{{Name: deviceName}}}},
},
},
},
expectedResourceSlices: []resourceapi.ResourceSlice{
*MakeResourceSlice().Name(generatedName1).GenerateName(generateName).
AppOwnerReferences(ownerName).NodeSelector(nodeSelector).
Driver(driverName).Devices([]resourceapi.Device{{Name: deviceName}}).
Pool(resourceapi.ResourcePool{Name: poolName, Generation: 1, ResourceSliceCount: 1}).Obj(),
},
},
"update-node-selector": {
initialObjects: []runtime.Object{
MakeResourceSlice().Name(resourceSlice1).UID(resourceSlice1).
AppOwnerReferences(ownerName).NodeSelector(nodeSelector).
Driver(driverName).Devices([]resourceapi.Device{{Name: deviceName}}).
Pool(resourceapi.ResourcePool{Name: poolName, Generation: 1, ResourceSliceCount: 1}).Obj(),
},
inputDriverResources: &DriverResources{
Pools: map[string]Pool{
poolName: {
NodeSelector: otherNodeSelector,
Slices: []Slice{{Devices: []resourceapi.Device{{Name: deviceName}}}},
},
},
},
expectedResourceSlices: []resourceapi.ResourceSlice{
*MakeResourceSlice().Name(resourceSlice1).UID(resourceSlice1).ResourceVersion("1").
AppOwnerReferences(ownerName).NodeSelector(otherNodeSelector).
Driver(driverName).Devices([]resourceapi.Device{{Name: deviceName}}).
Pool(resourceapi.ResourcePool{Name: poolName, Generation: 1, ResourceSliceCount: 1}).Obj(),
},
}, },
} }
for name, test := range testCases { for name, test := range testCases {
t.Run(name, func(t *testing.T) { t.Run(name, func(t *testing.T) {
_, ctx := ktesting.NewTestContext(t) _, ctx := ktesting.NewTestContext(t)
kubeClient := fake.NewSimpleClientset(test.initialObjects...) inputObjects := make([]runtime.Object, 0, len(test.initialObjects)+len(test.initialOtherObjects))
owner := Owner{ for _, initialOtherObject := range test.initialOtherObjects {
inputObjects = append(inputObjects, initialOtherObject.DeepCopyObject())
}
for _, initialObject := range test.initialObjects {
if _, ok := initialObject.(*resourceapi.ResourceSlice); !ok {
t.Fatalf("test.initialObjects have to be of type *resourceapi.ResourceSlice")
}
inputObjects = append(inputObjects, initialObject.DeepCopyObject())
}
kubeClient := createTestClient(inputObjects...)
var queue workqueue.Mock[string]
owner := &Owner{
APIVersion: "v1", APIVersion: "v1",
Kind: "Node", Kind: "Node",
Name: node, Name: ownerName,
UID: test.nodeUID, UID: test.nodeUID,
} }
ctrl, err := newController(ctx, kubeClient, driveName, owner, test.inputDriverResources) if test.nodeUID == "" {
owner = &Owner{
APIVersion: "apps/v1",
Kind: "Something",
Name: ownerName,
}
}
if test.noOwner {
owner = nil
}
ctrl, err := newController(ctx, Options{
DriverName: driverName,
KubeClient: kubeClient,
Owner: owner,
Resources: test.inputDriverResources,
Queue: &queue,
})
defer ctrl.Stop() defer ctrl.Stop()
require.NoError(t, err, "unexpected controller creation error") require.NoError(t, err, "unexpected controller creation error")
err = ctrl.syncPool(ctx, test.poolName)
require.NoError(t, err, "unexpected syncPool error") // Process work items in the queue until the queue is empty.
// Processing races with informers adding new work items,
// but the desired state should already be reached in the
// first iteration, so all following iterations should be nops.
ctrl.run(ctx)
// Check ResourceSlices // Check ResourceSlices
resourceSlices, err := kubeClient.ResourceV1alpha3().ResourceSlices().List(ctx, metav1.ListOptions{}) resourceSlices, err := kubeClient.ResourceV1alpha3().ResourceSlices().List(ctx, metav1.ListOptions{})
require.NoError(t, err, "list resource slices") require.NoError(t, err, "list resource slices")
assert.Equal(t, test.expectedResourceSlices, resourceSlices.Items, "unexpected ResourceSlices")
sortResourceSlices(test.expectedResourceSlices)
sortResourceSlices(resourceSlices.Items)
assert.Equal(t, test.expectedResourceSlices, resourceSlices.Items)
// The informer might have added a work item after ctrl.run returned.
state := queue.State()
state.Ready = nil
assert.Equal(t, workqueue.MockState[string]{}, state)
}) })
} }
} }
func newNode(name string, uid types.UID) *v1.Node { func sortResourceSlices(slices []resourceapi.ResourceSlice) {
return &v1.Node{ sort.Slice(slices, func(i, j int) bool {
ObjectMeta: metav1.ObjectMeta{ if len(slices[i].Name) == 0 && len(slices[j].Name) == 0 {
Name: name, return slices[i].ObjectMeta.GenerateName < slices[j].ObjectMeta.GenerateName
UID: uid, }
}, return slices[i].Name < slices[j].Name
})
}
func createTestClient(objects ...runtime.Object) *fake.Clientset {
fakeClient := fake.NewSimpleClientset(objects...)
fakeClient.PrependReactor("create", "resourceslices", createResourceSliceCreateReactor())
fakeClient.PrependReactor("update", "resourceslices", resourceSliceUpdateReactor)
return fakeClient
}
// createResourceSliceCreateReactor returns a function which
// implements the logic required for the GenerateName field to work when using
// the fake client. Add it with client.PrependReactor to your fake client.
func createResourceSliceCreateReactor() func(action k8stesting.Action) (handled bool, ret runtime.Object, err error) {
nameCounter := 0
var mutex sync.Mutex
return func(action k8stesting.Action) (handled bool, ret runtime.Object, err error) {
mutex.Lock()
defer mutex.Unlock()
resourceslice := action.(k8stesting.CreateAction).GetObject().(*resourceapi.ResourceSlice)
if resourceslice.Name == "" && resourceslice.GenerateName != "" {
resourceslice.Name = fmt.Sprintf("%s%d", resourceslice.GenerateName, nameCounter)
}
nameCounter++
return false, nil, nil
} }
} }
func newResourceSlice(name, nodeName, driveName, poolName string, poolGeneration int64) *resourceapi.ResourceSlice { // resourceSliceUpdateReactor implements the ResourceVersion bump for a fake client.
return &resourceapi.ResourceSlice{ func resourceSliceUpdateReactor(action k8stesting.Action) (handled bool, ret runtime.Object, err error) {
ObjectMeta: metav1.ObjectMeta{ resourceslice := action.(k8stesting.UpdateAction).GetObject().(*resourceapi.ResourceSlice)
Name: name, rev := 0
UID: types.UID(name), if resourceslice.ResourceVersion != "" {
}, oldRev, err := strconv.Atoi(resourceslice.ResourceVersion)
if err != nil {
return false, nil, fmt.Errorf("ResourceVersion %q should have been an int: %w", resourceslice.ResourceVersion, err)
}
rev = oldRev
}
rev++
resourceslice.ResourceVersion = fmt.Sprintf("%d", rev)
return false, nil, nil
}
// ResourceSliceWrapper wraps a ResourceSlice.
type ResourceSliceWrapper struct {
resourceapi.ResourceSlice
}
// MakeResourceSlice creates a wrapper for a ResourceSlice.
func MakeResourceSlice() *ResourceSliceWrapper {
return &ResourceSliceWrapper{
resourceapi.ResourceSlice{
ObjectMeta: metav1.ObjectMeta{},
Spec: resourceapi.ResourceSliceSpec{ Spec: resourceapi.ResourceSliceSpec{
NodeName: nodeName, Pool: resourceapi.ResourcePool{},
Driver: driveName, Devices: []resourceapi.Device{},
Pool: resourceapi.ResourcePool{
Name: poolName,
ResourceSliceCount: 1,
Generation: poolGeneration,
}, },
}, },
} }
} }
func newExpectResourceSlice(name, nodeName, nodeUID, driveName, poolName, deviceName string, generateName bool, poolGeneration int64) *resourceapi.ResourceSlice { // Obj returns the inner ResourceSlice.
resourceSlice := &resourceapi.ResourceSlice{ func (r *ResourceSliceWrapper) Obj() *resourceapi.ResourceSlice {
ObjectMeta: metav1.ObjectMeta{ return &r.ResourceSlice
OwnerReferences: []metav1.OwnerReference{ }
// Name sets the value of ResourceSlice.ObjectMeta.Name
func (r *ResourceSliceWrapper) Name(name string) *ResourceSliceWrapper {
r.ObjectMeta.Name = name
return r
}
// GenerateName sets the value of ResourceSlice.ObjectMeta.GenerateName
func (r *ResourceSliceWrapper) GenerateName(generateName string) *ResourceSliceWrapper {
r.ObjectMeta.GenerateName = generateName
return r
}
// UID sets the value of ResourceSlice.ObjectMeta.UID
func (r *ResourceSliceWrapper) UID(uid string) *ResourceSliceWrapper {
r.ObjectMeta.UID = types.UID(uid)
return r
}
// ResourceVersion sets the value of ResourceSlice.ObjectMeta.ResourceVersion
func (r *ResourceSliceWrapper) ResourceVersion(rev string) *ResourceSliceWrapper {
r.ObjectMeta.ResourceVersion = rev
return r
}
// NodeOwnerReferences sets the value of ResourceSlice.ObjectMeta.NodeOwnerReferences
// to a v1.Node
func (r *ResourceSliceWrapper) NodeOwnerReferences(nodeName, nodeUID string) *ResourceSliceWrapper {
r.ObjectMeta.OwnerReferences = []metav1.OwnerReference{
{ {
APIVersion: "v1", APIVersion: "v1",
Kind: "Node", Kind: "Node",
@@ -246,25 +742,56 @@ func newExpectResourceSlice(name, nodeName, nodeUID, driveName, poolName, device
UID: types.UID(nodeUID), UID: types.UID(nodeUID),
Controller: ptr.To(true), Controller: ptr.To(true),
}, },
},
},
Spec: resourceapi.ResourceSliceSpec{
NodeName: nodeName,
Driver: driveName,
Pool: resourceapi.ResourcePool{
Name: poolName,
ResourceSliceCount: 1,
Generation: poolGeneration,
},
Devices: []resourceapi.Device{{Name: deviceName}},
},
} }
return r
if generateName { }
resourceSlice.ObjectMeta.GenerateName = name
} else { // AppOwnerReferences sets the value of ResourceSlice.ObjectMeta.NodeOwnerReferences
resourceSlice.ObjectMeta.Name = name // to some fictional app controller resource
resourceSlice.ObjectMeta.UID = types.UID(name) func (r *ResourceSliceWrapper) AppOwnerReferences(appName string) *ResourceSliceWrapper {
} r.ObjectMeta.OwnerReferences = []metav1.OwnerReference{
return resourceSlice {
APIVersion: "apps/v1",
Kind: "Something",
Name: appName,
Controller: ptr.To(true),
},
}
return r
}
// Driver sets the value of ResourceSlice.Spec.Driver
func (r *ResourceSliceWrapper) Driver(driver string) *ResourceSliceWrapper {
r.Spec.Driver = driver
return r
}
// Pool sets the value of ResourceSlice.Spec.Pool
func (r *ResourceSliceWrapper) Pool(pool resourceapi.ResourcePool) *ResourceSliceWrapper {
r.Spec.Pool = pool
return r
}
// NodeName sets the value of ResourceSlice.Spec.NodeName
func (r *ResourceSliceWrapper) NodeName(nodeName string) *ResourceSliceWrapper {
r.Spec.NodeName = nodeName
return r
}
// NodeSelector sets the value of ResourceSlice.Spec.NodeSelector
func (r *ResourceSliceWrapper) NodeSelector(nodeSelector *v1.NodeSelector) *ResourceSliceWrapper {
r.Spec.NodeSelector = nodeSelector
return r
}
// AllNodes sets the value of ResourceSlice.Spec.AllNodes
func (r *ResourceSliceWrapper) AllNodes(allNodes bool) *ResourceSliceWrapper {
r.Spec.AllNodes = allNodes
return r
}
// Devices sets the value of ResourceSlice.Spec.Devices
func (r *ResourceSliceWrapper) Devices(devices []resourceapi.Device) *ResourceSliceWrapper {
r.Spec.Devices = devices
return r
} }

View File

@@ -73,9 +73,9 @@ func (in *Pool) DeepCopyInto(out *Pool) {
*out = new(v1.NodeSelector) *out = new(v1.NodeSelector)
(*in).DeepCopyInto(*out) (*in).DeepCopyInto(*out)
} }
if in.Devices != nil { if in.Slices != nil {
in, out := &in.Devices, &out.Devices in, out := &in.Slices, &out.Slices
*out = make([]v1alpha3.Device, len(*in)) *out = make([]Slice, len(*in))
for i := range *in { for i := range *in {
(*in)[i].DeepCopyInto(&(*out)[i]) (*in)[i].DeepCopyInto(&(*out)[i])
} }
@@ -92,3 +92,26 @@ func (in *Pool) DeepCopy() *Pool {
in.DeepCopyInto(out) in.DeepCopyInto(out)
return out return out
} }
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *Slice) DeepCopyInto(out *Slice) {
*out = *in
if in.Devices != nil {
in, out := &in.Devices, &out.Devices
*out = make([]v1alpha3.Device, len(*in))
for i := range *in {
(*in)[i].DeepCopyInto(&(*out)[i])
}
}
return
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Slice.
func (in *Slice) DeepCopy() *Slice {
if in == nil {
return nil
}
out := new(Slice)
in.DeepCopyInto(out)
return out
}