Introduce storagebackend.ConfigForResource

This is a Config specialized for a GroupResource.
It will support generating new resource-specific metrics.
This commit is contained in:
Mike Spreitzer 2021-08-29 01:06:12 -04:00
parent 5871321310
commit 85bcd243aa
23 changed files with 65 additions and 39 deletions

View File

@ -201,7 +201,7 @@ func (c LegacyRESTStorageProvider) NewLegacyRESTStorage(restOptionsGetter generi
serviceClusterIPAllocator, err := ipallocator.New(&serviceClusterIPRange, func(max int, rangeSpec string) (allocator.Interface, error) {
mem := allocator.NewAllocationMap(max, rangeSpec)
// TODO etcdallocator package to return a storage interface via the storageFactory
etcd, err := serviceallocator.NewEtcd(mem, "/ranges/serviceips", api.Resource("serviceipallocations"), serviceStorageConfig)
etcd, err := serviceallocator.NewEtcd(mem, "/ranges/serviceips", serviceStorageConfig.ForResource(api.Resource("serviceipallocations")))
if err != nil {
return nil, err
}
@ -220,7 +220,7 @@ func (c LegacyRESTStorageProvider) NewLegacyRESTStorage(restOptionsGetter generi
secondaryServiceClusterIPAllocator, err = ipallocator.New(&c.SecondaryServiceIPRange, func(max int, rangeSpec string) (allocator.Interface, error) {
mem := allocator.NewAllocationMap(max, rangeSpec)
// TODO etcdallocator package to return a storage interface via the storageFactory
etcd, err := serviceallocator.NewEtcd(mem, "/ranges/secondaryserviceips", api.Resource("serviceipallocations"), serviceStorageConfig)
etcd, err := serviceallocator.NewEtcd(mem, "/ranges/secondaryserviceips", serviceStorageConfig.ForResource(api.Resource("serviceipallocations")))
if err != nil {
return nil, err
}
@ -237,7 +237,7 @@ func (c LegacyRESTStorageProvider) NewLegacyRESTStorage(restOptionsGetter generi
serviceNodePortAllocator, err := portallocator.New(c.ServiceNodePortRange, func(max int, rangeSpec string) (allocator.Interface, error) {
mem := allocator.NewAllocationMap(max, rangeSpec)
// TODO etcdallocator package to return a storage interface via the storageFactory
etcd, err := serviceallocator.NewEtcd(mem, "/ranges/servicenodeports", api.Resource("servicenodeportallocations"), serviceStorageConfig)
etcd, err := serviceallocator.NewEtcd(mem, "/ranges/servicenodeports", serviceStorageConfig.ForResource(api.Resource("servicenodeportallocations")))
if err != nil {
return nil, err
}

View File

@ -40,7 +40,7 @@ func TestGetServersToValidate(t *testing.T) {
type fakeStorageFactory struct{}
func (f fakeStorageFactory) NewConfig(groupResource schema.GroupResource) (*storagebackend.Config, error) {
func (f fakeStorageFactory) NewConfig(groupResource schema.GroupResource) (*storagebackend.ConfigForResource, error) {
return nil, nil
}

View File

@ -61,7 +61,7 @@ var _ rangeallocation.RangeRegistry = &Etcd{}
// NewEtcd returns an allocator that is backed by Etcd and can manage
// persisting the snapshot state of allocation after each allocation is made.
func NewEtcd(alloc allocator.Snapshottable, baseKey string, resource schema.GroupResource, config *storagebackend.Config) (*Etcd, error) {
func NewEtcd(alloc allocator.Snapshottable, baseKey string, config *storagebackend.ConfigForResource) (*Etcd, error) {
storage, d, err := generic.NewRawStorage(config, nil)
if err != nil {
return nil, err
@ -76,7 +76,7 @@ func NewEtcd(alloc allocator.Snapshottable, baseKey string, resource schema.Grou
alloc: alloc,
storage: storage,
baseKey: baseKey,
resource: resource,
resource: config.GroupResource,
}, nil
}

View File

@ -32,11 +32,11 @@ import (
func newStorage(t *testing.T) (*Etcd, *etcd3testing.EtcdTestServer, allocator.Interface, *storagebackend.Config) {
etcdStorage, server := registrytest.NewEtcdStorage(t, "")
mem := allocator.NewAllocationMap(100, "rangeSpecValue")
etcd, err := NewEtcd(mem, "/ranges/serviceips", api.Resource("serviceipallocations"), etcdStorage)
etcd, err := NewEtcd(mem, "/ranges/serviceips", etcdStorage.ForResource(api.Resource("serviceipallocations")))
if err != nil {
t.Fatalf("unexpected error creating etcd: %v", err)
}
return etcd, server, mem, etcdStorage
return etcd, server, mem, &etcdStorage.Config
}
func validNewRangeAllocation() *api.RangeAllocation {
@ -95,7 +95,7 @@ func TestStore(t *testing.T) {
}
other = allocator.NewAllocationMap(100, "rangeSpecValue")
otherStorage, err := NewEtcd(other, "/ranges/serviceips", api.Resource("serviceipallocations"), config)
otherStorage, err := NewEtcd(other, "/ranges/serviceips", config.ForResource(api.Resource("serviceipallocations")))
if err != nil {
t.Fatalf("unexpected error creating etcd: %v", err)
}

View File

@ -42,10 +42,11 @@ func newStorage(t *testing.T) (*etcd3testing.EtcdTestServer, ipallocator.Interfa
}
var backing allocator.Interface
configForAllocations := etcdStorage.ForResource(api.Resource("serviceipallocations"))
storage, err := ipallocator.New(cidr, func(max int, rangeSpec string) (allocator.Interface, error) {
mem := allocator.NewAllocationMap(max, rangeSpec)
backing = mem
etcd, err := allocatorstore.NewEtcd(mem, "/ranges/serviceips", api.Resource("serviceipallocations"), etcdStorage)
etcd, err := allocatorstore.NewEtcd(mem, "/ranges/serviceips", configForAllocations)
if err != nil {
return nil, err
}
@ -54,7 +55,7 @@ func newStorage(t *testing.T) (*etcd3testing.EtcdTestServer, ipallocator.Interfa
if err != nil {
t.Fatalf("unexpected error creating etcd: %v", err)
}
s, d, err := generic.NewRawStorage(etcdStorage, nil)
s, d, err := generic.NewRawStorage(configForAllocations, nil)
if err != nil {
t.Fatalf("Couldn't create storage: %v", err)
}

View File

@ -44,11 +44,12 @@ func newStorage(t *testing.T) (*etcd3testing.EtcdTestServer, portallocator.Inter
etcdStorage, server := registrytest.NewEtcdStorage(t, "")
serviceNodePortRange := utilnet.PortRange{Base: basePortRange, Size: sizePortRange}
configForAllocations := etcdStorage.ForResource(api.Resource("servicenodeportallocations"))
var backing allocator.Interface
storage, err := portallocator.New(serviceNodePortRange, func(max int, rangeSpec string) (allocator.Interface, error) {
mem := allocator.NewAllocationMap(max, rangeSpec)
backing = mem
etcd, err := allocatorstore.NewEtcd(mem, "/ranges/servicenodeports", api.Resource("servicenodeportallocations"), etcdStorage)
etcd, err := allocatorstore.NewEtcd(mem, "/ranges/servicenodeports", configForAllocations)
if err != nil {
return nil, err
}
@ -57,7 +58,7 @@ func newStorage(t *testing.T) (*etcd3testing.EtcdTestServer, portallocator.Inter
if err != nil {
t.Fatalf("unexpected error creating etcd: %v", err)
}
s, d, err := generic.NewRawStorage(etcdStorage, nil)
s, d, err := generic.NewRawStorage(configForAllocations, nil)
if err != nil {
t.Fatalf("Couldn't create storage: %v", err)
}

View File

@ -29,6 +29,7 @@ import (
metainternalversion "k8s.io/apimachinery/pkg/apis/meta/internalversion"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/util/intstr"
utilnet "k8s.io/apimachinery/pkg/util/net"
"k8s.io/apimachinery/pkg/watch"
@ -175,7 +176,7 @@ func NewTestRESTWithPods(t *testing.T, endpoints []*api.Endpoints, pods []api.Po
serviceStorage := &serviceStorage{}
podStorage, err := podstore.NewStorage(generic.RESTOptions{
StorageConfig: etcdStorage,
StorageConfig: etcdStorage.ForResource(schema.GroupResource{Resource: "pods"}),
Decorator: generic.UndecoratedStorage,
DeleteCollectionWorkers: 3,
ResourcePrefix: "pods",
@ -191,7 +192,7 @@ func NewTestRESTWithPods(t *testing.T, endpoints []*api.Endpoints, pods []api.Po
}
}
endpointStorage, err := endpointstore.NewREST(generic.RESTOptions{
StorageConfig: etcdStorage,
StorageConfig: etcdStorage.ForResource(schema.GroupResource{Resource: "endpoints"}),
Decorator: generic.UndecoratedStorage,
ResourcePrefix: "endpoints",
})

View File

@ -24,6 +24,7 @@ import (
"k8s.io/apimachinery/pkg/fields"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/util/intstr"
"k8s.io/apiserver/pkg/registry/generic"
genericregistrytest "k8s.io/apiserver/pkg/registry/generic/testing"
@ -40,7 +41,7 @@ import (
func newStorage(t *testing.T) (*GenericREST, *StatusREST, *etcd3testing.EtcdTestServer) {
etcdStorage, server := registrytest.NewEtcdStorage(t, "")
restOptions := generic.RESTOptions{
StorageConfig: etcdStorage,
StorageConfig: etcdStorage.ForResource(schema.GroupResource{Resource: "services"}),
Decorator: generic.UndecoratedStorage,
DeleteCollectionWorkers: 1,
ResourcePrefix: "services",
@ -415,7 +416,7 @@ func TestServiceDefaultOnRead(t *testing.T) {
makeStorage := func(t *testing.T) (*GenericREST, *etcd3testing.EtcdTestServer) {
etcdStorage, server := registrytest.NewEtcdStorage(t, "")
restOptions := generic.RESTOptions{
StorageConfig: etcdStorage,
StorageConfig: etcdStorage.ForResource(schema.GroupResource{Resource: "services"}),
Decorator: generic.UndecoratedStorage,
DeleteCollectionWorkers: 1,
ResourcePrefix: "services",
@ -473,7 +474,7 @@ func TestServiceDefaulting(t *testing.T) {
makeStorage := func(t *testing.T, primaryCIDR string, isDualStack bool) (*GenericREST, *StatusREST, *etcd3testing.EtcdTestServer) {
etcdStorage, server := registrytest.NewEtcdStorage(t, "")
restOptions := generic.RESTOptions{
StorageConfig: etcdStorage,
StorageConfig: etcdStorage.ForResource(schema.GroupResource{Resource: "services"}),
Decorator: generic.UndecoratedStorage,
DeleteCollectionWorkers: 1,
ResourcePrefix: "services",

View File

@ -27,11 +27,12 @@ import (
"k8s.io/kubernetes/pkg/kubeapiserver"
)
func NewEtcdStorage(t *testing.T, group string) (*storagebackend.Config, *etcd3testing.EtcdTestServer) {
// NewEtcdStorage is for testing. It configures the etcd storage for a bogus resource; the test must not care.
func NewEtcdStorage(t *testing.T, group string) (*storagebackend.ConfigForResource, *etcd3testing.EtcdTestServer) {
return NewEtcdStorageForResource(t, schema.GroupResource{Group: group, Resource: "any"})
}
func NewEtcdStorageForResource(t *testing.T, resource schema.GroupResource) (*storagebackend.Config, *etcd3testing.EtcdTestServer) {
func NewEtcdStorageForResource(t *testing.T, resource schema.GroupResource) (*storagebackend.ConfigForResource, *etcd3testing.EtcdTestServer) {
t.Helper()
server, config := etcd3testing.NewUnsecuredEtcd3TestClientServer(t)

View File

@ -1147,7 +1147,7 @@ type CRDRESTOptionsGetter struct {
func (t CRDRESTOptionsGetter) GetRESTOptions(resource schema.GroupResource) (generic.RESTOptions, error) {
ret := generic.RESTOptions{
StorageConfig: &t.StorageConfig,
StorageConfig: t.StorageConfig.ForResource(resource),
Decorator: generic.UndecoratedStorage,
EnableGarbageCollection: t.EnableGarbageCollection,
DeleteCollectionWorkers: t.DeleteCollectionWorkers,

View File

@ -470,7 +470,7 @@ func testHandlerConversion(t *testing.T, enableWatchCache bool) {
etcdOptions := options.NewEtcdOptions(storageConfig)
etcdOptions.StorageConfig.Codec = unstructured.UnstructuredJSONScheme
restOptionsGetter := generic.RESTOptions{
StorageConfig: &etcdOptions.StorageConfig,
StorageConfig: etcdOptions.StorageConfig.ForResource(schema.GroupResource{Group: crd.Spec.Group, Resource: crd.Spec.Names.Plural}),
Decorator: generic.UndecoratedStorage,
EnableGarbageCollection: true,
DeleteCollectionWorkers: 1,

View File

@ -51,7 +51,8 @@ import (
func newStorage(t *testing.T) (customresource.CustomResourceStorage, *etcd3testing.EtcdTestServer) {
server, etcdStorage := etcd3testing.NewUnsecuredEtcd3TestClientServer(t)
etcdStorage.Codec = unstructured.UnstructuredJSONScheme
restOptions := generic.RESTOptions{StorageConfig: etcdStorage, Decorator: generic.UndecoratedStorage, DeleteCollectionWorkers: 1, ResourcePrefix: "noxus"}
groupResource := schema.GroupResource{Group: "mygroup.example.com", Resource: "noxus"}
restOptions := generic.RESTOptions{StorageConfig: etcdStorage.ForResource(groupResource), Decorator: generic.UndecoratedStorage, DeleteCollectionWorkers: 1, ResourcePrefix: "noxus"}
parameterScheme := runtime.NewScheme()
parameterScheme.AddUnversionedTypes(schema.GroupVersion{Group: "mygroup.example.com", Version: "v1beta1"},
@ -91,7 +92,7 @@ func newStorage(t *testing.T) (customresource.CustomResourceStorage, *etcd3testi
table, _ := tableconvertor.New(headers)
storage := customresource.NewStorage(
schema.GroupResource{Group: "mygroup.example.com", Resource: "noxus"},
groupResource,
kind,
schema.GroupVersionKind{Group: "mygroup.example.com", Version: "v1beta1", Kind: "NoxuItemList"},
customresource.NewStrategy(

View File

@ -26,9 +26,9 @@ import (
"k8s.io/client-go/tools/cache"
)
// RESTOptions is set of configuration options to generic registries.
// RESTOptions is set of resource-specific configuration options to generic registries.
type RESTOptions struct {
StorageConfig *storagebackend.Config
StorageConfig *storagebackend.ConfigForResource
Decorator StorageDecorator
EnableGarbageCollection bool

View File

@ -27,6 +27,7 @@ import (
"k8s.io/apimachinery/pkg/api/apitesting"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/types"
examplev1 "k8s.io/apiserver/pkg/apis/example/v1"
"k8s.io/apiserver/pkg/registry/rest"
@ -38,7 +39,7 @@ import (
func NewDryRunnableTestStorage(t *testing.T) (DryRunnableStorage, func()) {
server, sc := etcd3testing.NewUnsecuredEtcd3TestClientServer(t)
sc.Codec = apitesting.TestStorageCodec(codecs, examplev1.SchemeGroupVersion)
s, destroy, err := factory.Create(*sc, nil)
s, destroy, err := factory.Create(*sc.ForResource(schema.GroupResource{Resource: "pods"}), nil)
if err != nil {
t.Fatalf("Error creating storage: %v", err)
}

View File

@ -36,7 +36,7 @@ import (
// Creates a cacher based given storageConfig.
func StorageWithCacher() generic.StorageDecorator {
return func(
storageConfig *storagebackend.Config,
storageConfig *storagebackend.ConfigForResource,
resourcePrefix string,
keyFunc func(obj runtime.Object) (string, error),
newFunc func() runtime.Object,

View File

@ -2230,7 +2230,7 @@ func newTestGenericStoreRegistry(t *testing.T, scheme *runtime.Scheme, hasCacheE
newListFunc := func() runtime.Object { return &example.PodList{} }
sc.Codec = apitesting.TestStorageCodec(codecs, examplev1.SchemeGroupVersion)
s, dFunc, err := factory.Create(*sc, newFunc)
s, dFunc, err := factory.Create(*sc.ForResource(schema.GroupResource{Resource: "pods"}), newFunc)
if err != nil {
t.Fatalf("Error creating storage: %v", err)
}

View File

@ -27,7 +27,7 @@ import (
// StorageDecorator is a function signature for producing a storage.Interface
// and an associated DestroyFunc from given parameters.
type StorageDecorator func(
config *storagebackend.Config,
config *storagebackend.ConfigForResource,
resourcePrefix string,
keyFunc func(obj runtime.Object) (string, error),
newFunc func() runtime.Object,
@ -39,7 +39,7 @@ type StorageDecorator func(
// UndecoratedStorage returns the given a new storage from the given config
// without any decoration.
func UndecoratedStorage(
config *storagebackend.Config,
config *storagebackend.ConfigForResource,
resourcePrefix string,
keyFunc func(obj runtime.Object) (string, error),
newFunc func() runtime.Object,
@ -53,6 +53,6 @@ func UndecoratedStorage(
// NewRawStorage creates the low level kv storage. This is a work-around for current
// two layer of same storage interface.
// TODO: Once cacher is enabled on all registries (event registry is special), we will remove this method.
func NewRawStorage(config *storagebackend.Config, newFunc func() runtime.Object) (storage.Interface, factory.DestroyFunc, error) {
func NewRawStorage(config *storagebackend.ConfigForResource, newFunc func() runtime.Object) (storage.Interface, factory.DestroyFunc, error) {
return factory.Create(*config, newFunc)
}

View File

@ -255,7 +255,7 @@ type SimpleRestOptionsFactory struct {
func (f *SimpleRestOptionsFactory) GetRESTOptions(resource schema.GroupResource) (generic.RESTOptions, error) {
ret := generic.RESTOptions{
StorageConfig: &f.Options.StorageConfig,
StorageConfig: f.Options.StorageConfig.ForResource(resource),
Decorator: generic.UndecoratedStorage,
EnableGarbageCollection: f.Options.EnableGarbageCollection,
DeleteCollectionWorkers: f.Options.DeleteCollectionWorkers,

View File

@ -46,7 +46,7 @@ type Backend struct {
type StorageFactory interface {
// New finds the storage destination for the given group and resource. It will
// return an error if the group has no storage destination configured.
NewConfig(groupResource schema.GroupResource) (*storagebackend.Config, error)
NewConfig(groupResource schema.GroupResource) (*storagebackend.ConfigForResource, error)
// ResourcePrefix returns the overridden resource prefix for the GroupResource
// This allows for cohabitation of resources with different native types and provides
@ -250,7 +250,7 @@ func (s *DefaultStorageFactory) getStorageGroupResource(groupResource schema.Gro
// New finds the storage destination for the given group and resource. It will
// return an error if the group has no storage destination configured.
func (s *DefaultStorageFactory) NewConfig(groupResource schema.GroupResource) (*storagebackend.Config, error) {
func (s *DefaultStorageFactory) NewConfig(groupResource schema.GroupResource) (*storagebackend.ConfigForResource, error) {
chosenStorageResource := s.getStorageGroupResource(groupResource)
// operate on copy
@ -284,7 +284,7 @@ func (s *DefaultStorageFactory) NewConfig(groupResource schema.GroupResource) (*
}
klog.V(3).Infof("storing %v in %v, reading as %v from %#v", groupResource, codecConfig.StorageVersion, codecConfig.MemoryVersion, codecConfig.Config)
return &storageConfig, nil
return storageConfig.ForResource(groupResource), nil
}
// Backends returns all backends for all registered storage destinations.

View File

@ -21,6 +21,7 @@ import (
"go.opentelemetry.io/otel/trace"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apiserver/pkg/server/egressselector"
"k8s.io/apiserver/pkg/storage/etcd3"
"k8s.io/apiserver/pkg/storage/value"
@ -91,6 +92,23 @@ type Config struct {
StorageObjectCountTracker flowcontrolrequest.StorageObjectCountTracker
}
// ConfigForResource is a Config specialized to a particular `schema.GroupResource`
type ConfigForResource struct {
// Config is the resource-independent configuration
Config
// GroupResource is the relevant one
GroupResource schema.GroupResource
}
// ForResource specializes to the given resource
func (config *Config) ForResource(resource schema.GroupResource) *ConfigForResource {
return &ConfigForResource{
Config: *config,
GroupResource: resource,
}
}
func NewDefaultConfig(prefix string, codec runtime.Codec) *Config {
return &Config{
Paging: true,

View File

@ -244,7 +244,7 @@ func startCompactorOnce(c storagebackend.TransportConfig, interval time.Duration
}, nil
}
func newETCD3Storage(c storagebackend.Config, newFunc func() runtime.Object) (storage.Interface, DestroyFunc, error) {
func newETCD3Storage(c storagebackend.ConfigForResource, newFunc func() runtime.Object) (storage.Interface, DestroyFunc, error) {
stopCompactor, err := startCompactorOnce(c.Transport, c.CompactionInterval)
if err != nil {
return nil, nil, err

View File

@ -28,7 +28,7 @@ import (
type DestroyFunc func()
// Create creates a storage backend based on given config.
func Create(c storagebackend.Config, newFunc func() runtime.Object) (storage.Interface, DestroyFunc, error) {
func Create(c storagebackend.ConfigForResource, newFunc func() runtime.Object) (storage.Interface, DestroyFunc, error) {
switch c.Type {
case storagebackend.StorageTypeETCD2:
return nil, nil, fmt.Errorf("%s is no longer a supported storage backend", c.Type)

View File

@ -29,6 +29,7 @@ import (
apitesting "k8s.io/apimachinery/pkg/api/apitesting"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/runtime/serializer"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apiserver/pkg/apis/example"
@ -78,7 +79,7 @@ func TestTLSConnection(t *testing.T) {
},
Codec: codec,
}
storage, destroyFunc, err := newETCD3Storage(cfg, nil)
storage, destroyFunc, err := newETCD3Storage(*cfg.ForResource(schema.GroupResource{Resource: "pods"}), nil)
defer destroyFunc()
if err != nil {
t.Fatal(err)