mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-24 04:06:03 +00:00
Merge pull request #46800 from deads2k/agg-33-priority
Automatic merge from submit-queue (batch tested with PRs 47492, 47542, 46800, 47545, 45764) separate group and version priority Fixes https://github.com/kubernetes/kubernetes/issues/46322 This just modifies the API and does the minimal plumbing. I can extend this pull or do another to fix the priority problem.
This commit is contained in:
Kubernetes Submit Queue
GitHub
commit
08c705e875
@ -46746,7 +46746,8 @@
|
||||
"required": [
|
||||
"service",
|
||||
"caBundle",
|
||||
"priority"
|
||||
"groupPriorityMinimum",
|
||||
"versionPriority"
|
||||
],
|
||||
"properties": {
|
||||
"caBundle": {
|
||||
@ -46758,15 +46759,15 @@
|
||||
"description": "Group is the API group name this server hosts",
|
||||
"type": "string"
|
||||
},
|
||||
"groupPriorityMinimum": {
|
||||
"description": "GroupPriorityMininum is the priority this group should have at least. Higher priority means that the group is prefered by clients over lower priority ones. Note that other versions of this group might specify even higher GroupPriorityMininum values such that the whole group gets a higher priority. The primary sort is based on GroupPriorityMinimum, ordered highest number to lowest (20 before 10). The secondary sort is based on the alphabetical comparison of the name of the object. (v1.bar before v1.foo) We'd recommend something like: *.k8s.io (except extensions) at 18000 and PaaSes (OpenShift, Deis) are recommended to be in the 2000s",
|
||||
"type": "integer",
|
||||
"format": "int32"
|
||||
},
|
||||
"insecureSkipTLSVerify": {
|
||||
"description": "InsecureSkipTLSVerify disables TLS certificate verification when communicating with this server. This is strongly discouraged. You should use the CABundle instead.",
|
||||
"type": "boolean"
|
||||
},
|
||||
"priority": {
|
||||
"description": "Priority controls the ordering of this API group in the overall discovery document that gets served. Client tools like `kubectl` use this ordering to derive preference, so this ordering mechanism is important. Values must be between 1 and 1000 The primary sort is based on priority, ordered lowest number to highest (10 before 20). The secondary sort is based on the alphabetical comparison of the name of the object. (v1.bar before v1.foo) We'd recommend something like: *.k8s.io (except extensions) at 100, extensions at 150 PaaSes (OpenShift, Deis) are recommended to be in the 200s",
|
||||
"type": "integer",
|
||||
"format": "int64"
|
||||
},
|
||||
"service": {
|
||||
"description": "Service is a reference to the service for this API server. It must communicate on port 443 If the Service is nil, that means the handling for the API groupversion is handled locally on this server. The call will simply delegate to the normal handler chain to be fulfilled.",
|
||||
"$ref": "#/definitions/io.k8s.kube-aggregator.pkg.apis.apiregistration.v1beta1.ServiceReference"
|
||||
@ -46774,6 +46775,11 @@
|
||||
"version": {
|
||||
"description": "Version is the API version this server hosts. For example, \"v1\"",
|
||||
"type": "string"
|
||||
},
|
||||
"versionPriority": {
|
||||
"description": "VersionPriority controls the ordering of this API version inside of its group. Must be greater than zero. The primary sort is based on VersionPriority, ordered highest to lowest (20 before 10). The secondary sort is based on the alphabetical comparison of the name of the object. (v1.bar before v1.foo) Since it's inside of a group, the number can be small, probably in the 10s.",
|
||||
"type": "integer",
|
||||
"format": "int32"
|
||||
}
|
||||
}
|
||||
},
|
||||
|
||||
@ -25,6 +25,8 @@ import (
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"github.com/golang/glog"
|
||||
|
||||
apiextensionsinformers "k8s.io/apiextensions-apiserver/pkg/client/informers/internalversion"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/labels"
|
||||
@ -144,16 +146,59 @@ func createAggregatorServer(aggregatorConfig *aggregatorapiserver.Config, delega
|
||||
}
|
||||
|
||||
func makeAPIService(gv schema.GroupVersion) *apiregistration.APIService {
|
||||
apiServicePriority, ok := apiVersionPriorities[gv]
|
||||
if !ok {
|
||||
// if we aren't found, then we shouldn't register ourselves because it could result in a CRD group version
|
||||
// being permanently stuck in the APIServices list.
|
||||
glog.Infof("Skipping APIService creation for %v", gv)
|
||||
return nil
|
||||
}
|
||||
return &apiregistration.APIService{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: gv.Version + "." + gv.Group},
|
||||
Spec: apiregistration.APIServiceSpec{
|
||||
Group: gv.Group,
|
||||
Version: gv.Version,
|
||||
Priority: 100,
|
||||
Group: gv.Group,
|
||||
Version: gv.Version,
|
||||
GroupPriorityMinimum: apiServicePriority.group,
|
||||
VersionPriority: apiServicePriority.version,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
type priority struct {
|
||||
group int32
|
||||
version int32
|
||||
}
|
||||
|
||||
// The proper way to resolve this letting the aggregator know the desired group and version-within-group order of the underlying servers
|
||||
// is to refactor the genericapiserver.DelegationTarget to include a list of priorities based on which APIs were installed.
|
||||
// This requires the APIGroupInfo struct to evolve and include the concept of priorities and to avoid mistakes, the core storage map there needs to be updated.
|
||||
// That ripples out every bit as far as you'd expect, so for 1.7 we'll include the list here instead of being built up during storage.
|
||||
var apiVersionPriorities = map[schema.GroupVersion]priority{
|
||||
{Group: "", Version: "v1"}: {group: 18000, version: 1},
|
||||
// extensions is above the rest for CLI compatibility, though the level of unqalified resource compatibility we
|
||||
// can reasonably expect seems questionable.
|
||||
{Group: "extensions", Version: "v1beta1"}: {group: 17900, version: 1},
|
||||
// to my knowledge, nothing below here collides
|
||||
{Group: "apps", Version: "v1beta1"}: {group: 17800, version: 1},
|
||||
{Group: "authentication.k8s.io", Version: "v1"}: {group: 17700, version: 15},
|
||||
{Group: "authentication.k8s.io", Version: "v1beta1"}: {group: 17700, version: 9},
|
||||
{Group: "authorization.k8s.io", Version: "v1"}: {group: 17600, version: 15},
|
||||
{Group: "authorization.k8s.io", Version: "v1beta1"}: {group: 17600, version: 9},
|
||||
{Group: "autoscaling", Version: "v1"}: {group: 17500, version: 15},
|
||||
{Group: "autoscaling", Version: "v2alpha1"}: {group: 17500, version: 9},
|
||||
{Group: "batch", Version: "v1"}: {group: 17400, version: 15},
|
||||
{Group: "batch", Version: "v2alpha1"}: {group: 17400, version: 9},
|
||||
{Group: "certificates.k8s.io", Version: "v1beta1"}: {group: 17300, version: 9},
|
||||
{Group: "networking.k8s.io", Version: "v1"}: {group: 17200, version: 15},
|
||||
{Group: "policy", Version: "v1beta1"}: {group: 17100, version: 9},
|
||||
{Group: "rbac.authorization.k8s.io", Version: "v1beta1"}: {group: 17000, version: 12},
|
||||
{Group: "rbac.authorization.k8s.io", Version: "v1alpha1"}: {group: 17000, version: 9},
|
||||
{Group: "settings.k8s.io", Version: "v1alpha1"}: {group: 16900, version: 9},
|
||||
{Group: "storage.k8s.io", Version: "v1"}: {group: 16800, version: 15},
|
||||
{Group: "storage.k8s.io", Version: "v1beta1"}: {group: 16800, version: 9},
|
||||
{Group: "apiextensions.k8s.io", Version: "v1beta1"}: {group: 16700, version: 9},
|
||||
}
|
||||
|
||||
func apiServicesToRegister(delegateAPIServer genericapiserver.DelegationTarget, registration autoregister.AutoAPIServiceRegistration) []*apiregistration.APIService {
|
||||
apiServices := []*apiregistration.APIService{}
|
||||
|
||||
@ -175,14 +220,9 @@ func apiServicesToRegister(delegateAPIServer genericapiserver.DelegationTarget,
|
||||
}
|
||||
|
||||
apiService := makeAPIService(schema.GroupVersion{Group: tokens[2], Version: tokens[3]})
|
||||
|
||||
// TODO this is probably an indication that we need explicit and precise control over the discovery chain
|
||||
// but for now its a special case
|
||||
// apps has to come last for compatibility with 1.5 kubectl clients
|
||||
if apiService.Spec.Group == "apps" {
|
||||
apiService.Spec.Priority = 110
|
||||
if apiService == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
registration.AddAPIServiceToSync(apiService)
|
||||
apiServices = append(apiServices, apiService)
|
||||
}
|
||||
|
||||
@ -249,6 +249,10 @@ func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget,
|
||||
// The order here is preserved in discovery.
|
||||
// If resources with identical names exist in more than one of these groups (e.g. "deployments.apps"" and "deployments.extensions"),
|
||||
// the order of this list determines which group an unqualified resource name (e.g. "deployments") should prefer.
|
||||
// This priority order is used for local discovery, but it ends up aggregated in `k8s.io/kubernetes/cmd/kube-apiserver/app/aggregator.go
|
||||
// with specific priorities.
|
||||
// TODO: describe the priority all the way down in the RESTStorageProviders and plumb it back through the various discovery
|
||||
// handlers that we have.
|
||||
restStorageProviders := []RESTStorageProvider{
|
||||
authenticationrest.RESTStorageProvider{Authenticator: c.GenericConfig.Authenticator},
|
||||
authorizationrest.RESTStorageProvider{Authorizer: c.GenericConfig.Authorizer},
|
||||
|
||||
@ -255,9 +255,10 @@ func (c *tprRegistrationController) handleVersionUpdate(groupVersion schema.Grou
|
||||
c.apiServiceRegistration.AddAPIServiceToSync(&apiregistration.APIService{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: apiServiceName},
|
||||
Spec: apiregistration.APIServiceSpec{
|
||||
Group: groupVersion.Group,
|
||||
Version: groupVersion.Version,
|
||||
Priority: 500, // TPRs should have relatively low priority
|
||||
Group: groupVersion.Group,
|
||||
Version: groupVersion.Version,
|
||||
GroupPriorityMinimum: 1000, // TPRs should have relatively low priority
|
||||
VersionPriority: 100, // TPRs should have relatively low priority
|
||||
},
|
||||
})
|
||||
|
||||
|
||||
@ -84,9 +84,10 @@ func TestHandleVersionUpdate(t *testing.T) {
|
||||
{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "v1.group.com"},
|
||||
Spec: apiregistration.APIServiceSpec{
|
||||
Group: "group.com",
|
||||
Version: "v1",
|
||||
Priority: 500,
|
||||
Group: "group.com",
|
||||
Version: "v1",
|
||||
GroupPriorityMinimum: 1000,
|
||||
VersionPriority: 100,
|
||||
},
|
||||
},
|
||||
},
|
||||
@ -121,9 +122,10 @@ func TestHandleVersionUpdate(t *testing.T) {
|
||||
{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "v1.group.com"},
|
||||
Spec: apiregistration.APIServiceSpec{
|
||||
Group: "group.com",
|
||||
Version: "v1",
|
||||
Priority: 500,
|
||||
Group: "group.com",
|
||||
Version: "v1",
|
||||
GroupPriorityMinimum: 1000,
|
||||
VersionPriority: 100,
|
||||
},
|
||||
},
|
||||
},
|
||||
|
||||
@ -32,6 +32,8 @@ type RESTStorageProvider struct{}
|
||||
|
||||
func (p RESTStorageProvider) NewRESTStorage(apiResourceConfigSource serverstorage.APIResourceConfigSource, restOptionsGetter generic.RESTOptionsGetter) (genericapiserver.APIGroupInfo, bool) {
|
||||
apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(admissionregistration.GroupName, api.Registry, api.Scheme, api.ParameterCodec, api.Codecs)
|
||||
// If you add a version here, be sure to add an entry in `k8s.io/kubernetes/cmd/kube-apiserver/app/aggregator.go with specific priorities.
|
||||
// TODO refactor the plumbing to provide the information in the APIGroupInfo
|
||||
|
||||
if apiResourceConfigSource.AnyResourcesForVersionEnabled(admissionregistrationv1alpha1.SchemeGroupVersion) {
|
||||
apiGroupInfo.VersionedResourcesStorageMap[admissionregistrationv1alpha1.SchemeGroupVersion.Version] = p.v1alpha1Storage(apiResourceConfigSource, restOptionsGetter)
|
||||
|
||||
@ -33,6 +33,8 @@ type RESTStorageProvider struct{}
|
||||
|
||||
func (p RESTStorageProvider) NewRESTStorage(apiResourceConfigSource serverstorage.APIResourceConfigSource, restOptionsGetter generic.RESTOptionsGetter) (genericapiserver.APIGroupInfo, bool) {
|
||||
apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(apps.GroupName, api.Registry, api.Scheme, api.ParameterCodec, api.Codecs)
|
||||
// If you add a version here, be sure to add an entry in `k8s.io/kubernetes/cmd/kube-apiserver/app/aggregator.go with specific priorities.
|
||||
// TODO refactor the plumbing to provide the information in the APIGroupInfo
|
||||
|
||||
if apiResourceConfigSource.AnyResourcesForVersionEnabled(appsapiv1beta1.SchemeGroupVersion) {
|
||||
apiGroupInfo.VersionedResourcesStorageMap[appsapiv1beta1.SchemeGroupVersion.Version] = p.v1beta1Storage(apiResourceConfigSource, restOptionsGetter)
|
||||
|
||||
@ -40,6 +40,8 @@ func (p RESTStorageProvider) NewRESTStorage(apiResourceConfigSource serverstorag
|
||||
// }
|
||||
|
||||
apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(authentication.GroupName, api.Registry, api.Scheme, api.ParameterCodec, api.Codecs)
|
||||
// If you add a version here, be sure to add an entry in `k8s.io/kubernetes/cmd/kube-apiserver/app/aggregator.go with specific priorities.
|
||||
// TODO refactor the plumbing to provide the information in the APIGroupInfo
|
||||
|
||||
if apiResourceConfigSource.AnyResourcesForVersionEnabled(authenticationv1beta1.SchemeGroupVersion) {
|
||||
apiGroupInfo.VersionedResourcesStorageMap[authenticationv1beta1.SchemeGroupVersion.Version] = p.v1beta1Storage(apiResourceConfigSource, restOptionsGetter)
|
||||
|
||||
@ -41,6 +41,8 @@ func (p RESTStorageProvider) NewRESTStorage(apiResourceConfigSource serverstorag
|
||||
}
|
||||
|
||||
apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(authorization.GroupName, api.Registry, api.Scheme, api.ParameterCodec, api.Codecs)
|
||||
// If you add a version here, be sure to add an entry in `k8s.io/kubernetes/cmd/kube-apiserver/app/aggregator.go with specific priorities.
|
||||
// TODO refactor the plumbing to provide the information in the APIGroupInfo
|
||||
|
||||
if apiResourceConfigSource.AnyResourcesForVersionEnabled(authorizationv1beta1.SchemeGroupVersion) {
|
||||
apiGroupInfo.VersionedResourcesStorageMap[authorizationv1beta1.SchemeGroupVersion.Version] = p.v1beta1Storage(apiResourceConfigSource, restOptionsGetter)
|
||||
|
||||
@ -32,6 +32,8 @@ type RESTStorageProvider struct{}
|
||||
|
||||
func (p RESTStorageProvider) NewRESTStorage(apiResourceConfigSource serverstorage.APIResourceConfigSource, restOptionsGetter generic.RESTOptionsGetter) (genericapiserver.APIGroupInfo, bool) {
|
||||
apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(autoscaling.GroupName, api.Registry, api.Scheme, api.ParameterCodec, api.Codecs)
|
||||
// If you add a version here, be sure to add an entry in `k8s.io/kubernetes/cmd/kube-apiserver/app/aggregator.go with specific priorities.
|
||||
// TODO refactor the plumbing to provide the information in the APIGroupInfo
|
||||
|
||||
if apiResourceConfigSource.AnyResourcesForVersionEnabled(autoscalingapiv2alpha1.SchemeGroupVersion) {
|
||||
apiGroupInfo.VersionedResourcesStorageMap[autoscalingapiv2alpha1.SchemeGroupVersion.Version] = p.v2alpha1Storage(apiResourceConfigSource, restOptionsGetter)
|
||||
|
||||
@ -34,6 +34,8 @@ type RESTStorageProvider struct{}
|
||||
|
||||
func (p RESTStorageProvider) NewRESTStorage(apiResourceConfigSource serverstorage.APIResourceConfigSource, restOptionsGetter generic.RESTOptionsGetter) (genericapiserver.APIGroupInfo, bool) {
|
||||
apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(batch.GroupName, api.Registry, api.Scheme, api.ParameterCodec, api.Codecs)
|
||||
// If you add a version here, be sure to add an entry in `k8s.io/kubernetes/cmd/kube-apiserver/app/aggregator.go with specific priorities.
|
||||
// TODO refactor the plumbing to provide the information in the APIGroupInfo
|
||||
|
||||
if apiResourceConfigSource.AnyResourcesForVersionEnabled(batchapiv2alpha1.SchemeGroupVersion) {
|
||||
apiGroupInfo.VersionedResourcesStorageMap[batchapiv2alpha1.SchemeGroupVersion.Version] = p.v2alpha1Storage(apiResourceConfigSource, restOptionsGetter)
|
||||
|
||||
@ -31,6 +31,8 @@ type RESTStorageProvider struct{}
|
||||
|
||||
func (p RESTStorageProvider) NewRESTStorage(apiResourceConfigSource serverstorage.APIResourceConfigSource, restOptionsGetter generic.RESTOptionsGetter) (genericapiserver.APIGroupInfo, bool) {
|
||||
apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(certificates.GroupName, api.Registry, api.Scheme, api.ParameterCodec, api.Codecs)
|
||||
// If you add a version here, be sure to add an entry in `k8s.io/kubernetes/cmd/kube-apiserver/app/aggregator.go with specific priorities.
|
||||
// TODO refactor the plumbing to provide the information in the APIGroupInfo
|
||||
|
||||
if apiResourceConfigSource.AnyResourcesForVersionEnabled(certificatesapiv1beta1.SchemeGroupVersion) {
|
||||
apiGroupInfo.VersionedResourcesStorageMap[certificatesapiv1beta1.SchemeGroupVersion.Version] = p.v1beta1Storage(apiResourceConfigSource, restOptionsGetter)
|
||||
|
||||
@ -48,6 +48,8 @@ type RESTStorageProvider struct {
|
||||
|
||||
func (p RESTStorageProvider) NewRESTStorage(apiResourceConfigSource serverstorage.APIResourceConfigSource, restOptionsGetter generic.RESTOptionsGetter) (genericapiserver.APIGroupInfo, bool) {
|
||||
apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(extensions.GroupName, api.Registry, api.Scheme, api.ParameterCodec, api.Codecs)
|
||||
// If you add a version here, be sure to add an entry in `k8s.io/kubernetes/cmd/kube-apiserver/app/aggregator.go with specific priorities.
|
||||
// TODO refactor the plumbing to provide the information in the APIGroupInfo
|
||||
|
||||
if apiResourceConfigSource.AnyResourcesForVersionEnabled(extensionsapiv1beta1.SchemeGroupVersion) {
|
||||
apiGroupInfo.VersionedResourcesStorageMap[extensionsapiv1beta1.SchemeGroupVersion.Version] = p.v1beta1Storage(apiResourceConfigSource, restOptionsGetter)
|
||||
|
||||
@ -31,6 +31,8 @@ type RESTStorageProvider struct{}
|
||||
|
||||
func (p RESTStorageProvider) NewRESTStorage(apiResourceConfigSource serverstorage.APIResourceConfigSource, restOptionsGetter generic.RESTOptionsGetter) (genericapiserver.APIGroupInfo, bool) {
|
||||
apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(networking.GroupName, api.Registry, api.Scheme, api.ParameterCodec, api.Codecs)
|
||||
// If you add a version here, be sure to add an entry in `k8s.io/kubernetes/cmd/kube-apiserver/app/aggregator.go with specific priorities.
|
||||
// TODO refactor the plumbing to provide the information in the APIGroupInfo
|
||||
|
||||
if apiResourceConfigSource.AnyResourcesForVersionEnabled(networkingapiv1.SchemeGroupVersion) {
|
||||
apiGroupInfo.VersionedResourcesStorageMap[networkingapiv1.SchemeGroupVersion.Version] = p.v1alpha1Storage(apiResourceConfigSource, restOptionsGetter)
|
||||
|
||||
@ -31,6 +31,8 @@ type RESTStorageProvider struct{}
|
||||
|
||||
func (p RESTStorageProvider) NewRESTStorage(apiResourceConfigSource serverstorage.APIResourceConfigSource, restOptionsGetter generic.RESTOptionsGetter) (genericapiserver.APIGroupInfo, bool) {
|
||||
apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(policy.GroupName, api.Registry, api.Scheme, api.ParameterCodec, api.Codecs)
|
||||
// If you add a version here, be sure to add an entry in `k8s.io/kubernetes/cmd/kube-apiserver/app/aggregator.go with specific priorities.
|
||||
// TODO refactor the plumbing to provide the information in the APIGroupInfo
|
||||
|
||||
if apiResourceConfigSource.AnyResourcesForVersionEnabled(policyapiv1beta1.SchemeGroupVersion) {
|
||||
apiGroupInfo.VersionedResourcesStorageMap[policyapiv1beta1.SchemeGroupVersion.Version] = p.v1beta1Storage(apiResourceConfigSource, restOptionsGetter)
|
||||
|
||||
@ -65,6 +65,8 @@ var _ genericapiserver.PostStartHookProvider = RESTStorageProvider{}
|
||||
|
||||
func (p RESTStorageProvider) NewRESTStorage(apiResourceConfigSource serverstorage.APIResourceConfigSource, restOptionsGetter generic.RESTOptionsGetter) (genericapiserver.APIGroupInfo, bool) {
|
||||
apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(rbac.GroupName, api.Registry, api.Scheme, api.ParameterCodec, api.Codecs)
|
||||
// If you add a version here, be sure to add an entry in `k8s.io/kubernetes/cmd/kube-apiserver/app/aggregator.go with specific priorities.
|
||||
// TODO refactor the plumbing to provide the information in the APIGroupInfo
|
||||
|
||||
if apiResourceConfigSource.AnyResourcesForVersionEnabled(rbacapiv1alpha1.SchemeGroupVersion) {
|
||||
apiGroupInfo.VersionedResourcesStorageMap[rbacapiv1alpha1.SchemeGroupVersion.Version] = p.storage(rbacapiv1alpha1.SchemeGroupVersion, apiResourceConfigSource, restOptionsGetter)
|
||||
|
||||
@ -31,6 +31,8 @@ type RESTStorageProvider struct{}
|
||||
|
||||
func (p RESTStorageProvider) NewRESTStorage(apiResourceConfigSource serverstorage.APIResourceConfigSource, restOptionsGetter generic.RESTOptionsGetter) (genericapiserver.APIGroupInfo, bool) {
|
||||
apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(settings.GroupName, api.Registry, api.Scheme, api.ParameterCodec, api.Codecs)
|
||||
// If you add a version here, be sure to add an entry in `k8s.io/kubernetes/cmd/kube-apiserver/app/aggregator.go with specific priorities.
|
||||
// TODO refactor the plumbing to provide the information in the APIGroupInfo
|
||||
|
||||
if apiResourceConfigSource.AnyResourcesForVersionEnabled(settingsapiv1alpha1.SchemeGroupVersion) {
|
||||
apiGroupInfo.VersionedResourcesStorageMap[settingsapiv1alpha1.SchemeGroupVersion.Version] = p.v1alpha1Storage(apiResourceConfigSource, restOptionsGetter)
|
||||
|
||||
@ -33,6 +33,8 @@ type RESTStorageProvider struct {
|
||||
|
||||
func (p RESTStorageProvider) NewRESTStorage(apiResourceConfigSource serverstorage.APIResourceConfigSource, restOptionsGetter generic.RESTOptionsGetter) (genericapiserver.APIGroupInfo, bool) {
|
||||
apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(storageapi.GroupName, api.Registry, api.Scheme, api.ParameterCodec, api.Codecs)
|
||||
// If you add a version here, be sure to add an entry in `k8s.io/kubernetes/cmd/kube-apiserver/app/aggregator.go with specific priorities.
|
||||
// TODO refactor the plumbing to provide the information in the APIGroupInfo
|
||||
|
||||
if apiResourceConfigSource.AnyResourcesForVersionEnabled(storageapiv1beta1.SchemeGroupVersion) {
|
||||
apiGroupInfo.VersionedResourcesStorageMap[storageapiv1beta1.SchemeGroupVersion.Version] = p.v1beta1Storage(apiResourceConfigSource, restOptionsGetter)
|
||||
|
||||
@ -23,12 +23,12 @@ import (
|
||||
"k8s.io/apimachinery/pkg/runtime/schema"
|
||||
)
|
||||
|
||||
func SortedByGroup(servers []*APIService) [][]*APIService {
|
||||
serversByPriority := ByPriority(servers)
|
||||
sort.Sort(serversByPriority)
|
||||
func SortedByGroupAndVersion(servers []*APIService) [][]*APIService {
|
||||
serversByGroupPriorityMinimum := ByGroupPriorityMinimum(servers)
|
||||
sort.Sort(serversByGroupPriorityMinimum)
|
||||
|
||||
ret := [][]*APIService{}
|
||||
for _, curr := range serversByPriority {
|
||||
for _, curr := range serversByGroupPriorityMinimum {
|
||||
// check to see if we already have an entry for this group
|
||||
existingIndex := -1
|
||||
for j, groupInReturn := range ret {
|
||||
@ -40,6 +40,7 @@ func SortedByGroup(servers []*APIService) [][]*APIService {
|
||||
|
||||
if existingIndex >= 0 {
|
||||
ret[existingIndex] = append(ret[existingIndex], curr)
|
||||
sort.Sort(ByVersionPriority(ret[existingIndex]))
|
||||
continue
|
||||
}
|
||||
|
||||
@ -49,15 +50,32 @@ func SortedByGroup(servers []*APIService) [][]*APIService {
|
||||
return ret
|
||||
}
|
||||
|
||||
type ByPriority []*APIService
|
||||
// ByGroupPriorityMinimum sorts with the highest group number first, then by name.
|
||||
// This is not a simple reverse, because we want the name sorting to be alpha, not
|
||||
// reverse alpha.
|
||||
type ByGroupPriorityMinimum []*APIService
|
||||
|
||||
func (s ByPriority) Len() int { return len(s) }
|
||||
func (s ByPriority) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
|
||||
func (s ByPriority) Less(i, j int) bool {
|
||||
if s[i].Spec.Priority == s[j].Spec.Priority {
|
||||
return s[i].Name < s[j].Name
|
||||
func (s ByGroupPriorityMinimum) Len() int { return len(s) }
|
||||
func (s ByGroupPriorityMinimum) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
|
||||
func (s ByGroupPriorityMinimum) Less(i, j int) bool {
|
||||
if s[i].Spec.GroupPriorityMinimum != s[j].Spec.GroupPriorityMinimum {
|
||||
return s[i].Spec.GroupPriorityMinimum > s[j].Spec.GroupPriorityMinimum
|
||||
}
|
||||
return s[i].Spec.Priority < s[j].Spec.Priority
|
||||
return s[i].Name < s[j].Name
|
||||
}
|
||||
|
||||
// ByVersionPriority sorts with the highest version number first, then by name.
|
||||
// This is not a simple reverse, because we want the name sorting to be alpha, not
|
||||
// reverse alpha.
|
||||
type ByVersionPriority []*APIService
|
||||
|
||||
func (s ByVersionPriority) Len() int { return len(s) }
|
||||
func (s ByVersionPriority) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
|
||||
func (s ByVersionPriority) Less(i, j int) bool {
|
||||
if s[i].Spec.VersionPriority != s[j].Spec.VersionPriority {
|
||||
return s[i].Spec.VersionPriority > s[j].Spec.VersionPriority
|
||||
}
|
||||
return s[i].Name < s[j].Name
|
||||
}
|
||||
|
||||
// APIServiceNameToGroupVersion returns the GroupVersion for a given apiServiceName. The name
|
||||
|
||||
@ -53,14 +53,19 @@ type APIServiceSpec struct {
|
||||
// CABundle is a PEM encoded CA bundle which will be used to validate an API server's serving certificate.
|
||||
CABundle []byte
|
||||
|
||||
// Priority controls the ordering of this API group in the overall discovery document that gets served.
|
||||
// Client tools like `kubectl` use this ordering to derive preference, so this ordering mechanism is important.
|
||||
// Values must be between 1 and 1000
|
||||
// The primary sort is based on priority, ordered lowest number to highest (10 before 20).
|
||||
// GroupPriorityMininum is the priority this group should have at least. Higher priority means that the group is prefered by clients over lower priority ones.
|
||||
// Note that other versions of this group might specify even higher GroupPriorityMininum values such that the whole group gets a higher priority.
|
||||
// The primary sort is based on GroupPriorityMinimum, ordered highest number to lowest (20 before 10).
|
||||
// The secondary sort is based on the alphabetical comparison of the name of the object. (v1.bar before v1.foo)
|
||||
// We'd recommend something like: *.k8s.io (except extensions) at 100, extensions at 150
|
||||
// PaaSes (OpenShift, Deis) are recommended to be in the 200s
|
||||
Priority int64
|
||||
// We'd recommend something like: *.k8s.io (except extensions) at 18000 and
|
||||
// PaaSes (OpenShift, Deis) are recommended to be in the 2000s
|
||||
GroupPriorityMinimum int32
|
||||
|
||||
// VersionPriority controls the ordering of this API version inside of its group. Must be greater than zero.
|
||||
// The primary sort is based on VersionPriority, ordered highest to lowest (20 before 10).
|
||||
// The secondary sort is based on the alphabetical comparison of the name of the object. (v1.bar before v1.foo)
|
||||
// Since it's inside of a group, the number can be small, probably in the 10s.
|
||||
VersionPriority int32
|
||||
}
|
||||
|
||||
type ConditionStatus string
|
||||
|
||||
@ -255,9 +255,12 @@ func (m *APIServiceSpec) MarshalTo(dAtA []byte) (int, error) {
|
||||
i = encodeVarintGenerated(dAtA, i, uint64(len(m.CABundle)))
|
||||
i += copy(dAtA[i:], m.CABundle)
|
||||
}
|
||||
dAtA[i] = 0x30
|
||||
dAtA[i] = 0x38
|
||||
i++
|
||||
i = encodeVarintGenerated(dAtA, i, uint64(m.Priority))
|
||||
i = encodeVarintGenerated(dAtA, i, uint64(m.GroupPriorityMinimum))
|
||||
dAtA[i] = 0x40
|
||||
i++
|
||||
i = encodeVarintGenerated(dAtA, i, uint64(m.VersionPriority))
|
||||
return i, nil
|
||||
}
|
||||
|
||||
@ -402,7 +405,8 @@ func (m *APIServiceSpec) Size() (n int) {
|
||||
l = len(m.CABundle)
|
||||
n += 1 + l + sovGenerated(uint64(l))
|
||||
}
|
||||
n += 1 + sovGenerated(uint64(m.Priority))
|
||||
n += 1 + sovGenerated(uint64(m.GroupPriorityMinimum))
|
||||
n += 1 + sovGenerated(uint64(m.VersionPriority))
|
||||
return n
|
||||
}
|
||||
|
||||
@ -488,7 +492,8 @@ func (this *APIServiceSpec) String() string {
|
||||
`Version:` + fmt.Sprintf("%v", this.Version) + `,`,
|
||||
`InsecureSkipTLSVerify:` + fmt.Sprintf("%v", this.InsecureSkipTLSVerify) + `,`,
|
||||
`CABundle:` + valueToStringGenerated(this.CABundle) + `,`,
|
||||
`Priority:` + fmt.Sprintf("%v", this.Priority) + `,`,
|
||||
`GroupPriorityMinimum:` + fmt.Sprintf("%v", this.GroupPriorityMinimum) + `,`,
|
||||
`VersionPriority:` + fmt.Sprintf("%v", this.VersionPriority) + `,`,
|
||||
`}`,
|
||||
}, "")
|
||||
return s
|
||||
@ -1140,11 +1145,11 @@ func (m *APIServiceSpec) Unmarshal(dAtA []byte) error {
|
||||
m.CABundle = []byte{}
|
||||
}
|
||||
iNdEx = postIndex
|
||||
case 6:
|
||||
case 7:
|
||||
if wireType != 0 {
|
||||
return fmt.Errorf("proto: wrong wireType = %d for field Priority", wireType)
|
||||
return fmt.Errorf("proto: wrong wireType = %d for field GroupPriorityMinimum", wireType)
|
||||
}
|
||||
m.Priority = 0
|
||||
m.GroupPriorityMinimum = 0
|
||||
for shift := uint(0); ; shift += 7 {
|
||||
if shift >= 64 {
|
||||
return ErrIntOverflowGenerated
|
||||
@ -1154,7 +1159,26 @@ func (m *APIServiceSpec) Unmarshal(dAtA []byte) error {
|
||||
}
|
||||
b := dAtA[iNdEx]
|
||||
iNdEx++
|
||||
m.Priority |= (int64(b) & 0x7F) << shift
|
||||
m.GroupPriorityMinimum |= (int32(b) & 0x7F) << shift
|
||||
if b < 0x80 {
|
||||
break
|
||||
}
|
||||
}
|
||||
case 8:
|
||||
if wireType != 0 {
|
||||
return fmt.Errorf("proto: wrong wireType = %d for field VersionPriority", wireType)
|
||||
}
|
||||
m.VersionPriority = 0
|
||||
for shift := uint(0); ; shift += 7 {
|
||||
if shift >= 64 {
|
||||
return ErrIntOverflowGenerated
|
||||
}
|
||||
if iNdEx >= l {
|
||||
return io.ErrUnexpectedEOF
|
||||
}
|
||||
b := dAtA[iNdEx]
|
||||
iNdEx++
|
||||
m.VersionPriority |= (int32(b) & 0x7F) << shift
|
||||
if b < 0x80 {
|
||||
break
|
||||
}
|
||||
@ -1479,56 +1503,58 @@ func init() {
|
||||
}
|
||||
|
||||
var fileDescriptorGenerated = []byte{
|
||||
// 801 bytes of a gzipped FileDescriptorProto
|
||||
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x54, 0xcf, 0x6f, 0x1b, 0x45,
|
||||
0x14, 0xf6, 0x3a, 0xfe, 0x95, 0x49, 0x68, 0xc3, 0xa0, 0x0a, 0xcb, 0x12, 0xeb, 0x60, 0x24, 0x70,
|
||||
0x11, 0xdd, 0x25, 0x05, 0xf1, 0x43, 0x9c, 0xba, 0x39, 0x54, 0x96, 0x5c, 0xa8, 0xc6, 0x51, 0x0f,
|
||||
0x08, 0x89, 0x8e, 0xd7, 0x2f, 0xeb, 0xc1, 0xd9, 0x1f, 0xcc, 0xcc, 0x5a, 0xf2, 0xad, 0x7f, 0x02,
|
||||
0x17, 0xfe, 0xa7, 0x1c, 0x38, 0xf4, 0x98, 0x93, 0x45, 0xcc, 0x7f, 0xd1, 0x0b, 0x68, 0x66, 0x67,
|
||||
0x77, 0x5d, 0xdb, 0x15, 0x51, 0x94, 0x8b, 0xe5, 0x79, 0xef, 0x7d, 0xdf, 0xf7, 0xe6, 0xbd, 0x6f,
|
||||
0x07, 0xbd, 0x9c, 0x7d, 0x27, 0x1c, 0x16, 0xbb, 0xb3, 0x74, 0x0c, 0x3c, 0x02, 0x09, 0xc2, 0x9d,
|
||||
0x43, 0x34, 0x89, 0xb9, 0xbb, 0x96, 0x78, 0x44, 0x83, 0x80, 0x43, 0x40, 0x65, 0xcc, 0xdd, 0x64,
|
||||
0x16, 0xb8, 0x34, 0x61, 0x42, 0xfd, 0x70, 0x08, 0x98, 0x90, 0x9c, 0x4a, 0x16, 0x47, 0xee, 0xfc,
|
||||
0x64, 0x0c, 0x92, 0x9e, 0xb8, 0x01, 0x44, 0xc0, 0xa9, 0x84, 0x89, 0x93, 0xf0, 0x58, 0xc6, 0xf8,
|
||||
0xdb, 0x8c, 0xc8, 0x51, 0x44, 0xbf, 0x96, 0x44, 0x4e, 0x32, 0x0b, 0x1c, 0x45, 0xe4, 0x6c, 0x10,
|
||||
0x39, 0x86, 0xa8, 0xf3, 0x28, 0x60, 0x72, 0x9a, 0x8e, 0x1d, 0x3f, 0x0e, 0xdd, 0x20, 0x0e, 0x62,
|
||||
0x57, 0xf3, 0x8d, 0xd3, 0x73, 0x7d, 0xd2, 0x07, 0xfd, 0x2f, 0xd3, 0xe9, 0x7c, 0x6d, 0x1a, 0xa6,
|
||||
0x09, 0x0b, 0xa9, 0x3f, 0x65, 0x11, 0xf0, 0x45, 0xd9, 0x6d, 0x08, 0x92, 0xba, 0xf3, 0xad, 0xee,
|
||||
0x3a, 0xee, 0xbb, 0x50, 0x3c, 0x8d, 0x24, 0x0b, 0x61, 0x0b, 0xf0, 0xcd, 0xff, 0x01, 0x84, 0x3f,
|
||||
0x85, 0x90, 0x6e, 0xe1, 0xbe, 0x7a, 0x17, 0x2e, 0x95, 0xec, 0xc2, 0x65, 0x91, 0x14, 0x92, 0x6f,
|
||||
0x82, 0x7a, 0x7f, 0x55, 0x11, 0x7a, 0xf2, 0x7c, 0x30, 0x02, 0x3e, 0x67, 0x3e, 0xe0, 0x97, 0xa8,
|
||||
0xa5, 0xee, 0x31, 0xa1, 0x92, 0xb6, 0xad, 0x63, 0xab, 0x7f, 0xf0, 0xf8, 0x4b, 0xc7, 0x4c, 0x77,
|
||||
0x9d, 0xb6, 0x1c, 0xad, 0xaa, 0x76, 0xe6, 0x27, 0xce, 0x4f, 0xe3, 0xdf, 0xc0, 0x97, 0xcf, 0x40,
|
||||
0x52, 0x0f, 0x5f, 0x2e, 0xbb, 0x95, 0xd5, 0xb2, 0x8b, 0xca, 0x18, 0x29, 0x58, 0x31, 0x43, 0x35,
|
||||
0x91, 0x80, 0xdf, 0xae, 0x6a, 0xf6, 0xa7, 0xce, 0x2d, 0x77, 0xe7, 0x94, 0x4d, 0x8f, 0x12, 0xf0,
|
||||
0xbd, 0x43, 0x23, 0x5a, 0x53, 0x27, 0xa2, 0x25, 0xf0, 0xef, 0xa8, 0x21, 0x24, 0x95, 0xa9, 0x68,
|
||||
0xef, 0x69, 0xb1, 0xc1, 0x5d, 0x88, 0x69, 0x42, 0xef, 0x9e, 0x91, 0x6b, 0x64, 0x67, 0x62, 0x84,
|
||||
0x7a, 0x57, 0x55, 0xf4, 0x41, 0x59, 0x7c, 0x1a, 0x47, 0x13, 0xa6, 0x88, 0xf0, 0x0f, 0xa8, 0x26,
|
||||
0x17, 0x09, 0xe8, 0x99, 0xee, 0x7b, 0x9f, 0xe5, 0xcd, 0x9e, 0x2d, 0x12, 0x78, 0xb3, 0xec, 0x7e,
|
||||
0xb8, 0x03, 0xa2, 0x52, 0x44, 0x83, 0xf0, 0xf7, 0xc5, 0x3d, 0xaa, 0x1a, 0xfe, 0xf1, 0xdb, 0xe2,
|
||||
0x6f, 0x96, 0xdd, 0xfb, 0x05, 0xec, 0xed, 0x7e, 0xf0, 0x1c, 0xe1, 0x0b, 0x2a, 0xe4, 0x19, 0xa7,
|
||||
0x91, 0xc8, 0x68, 0x59, 0x08, 0x66, 0x1c, 0x9f, 0xdf, 0x6c, 0xb3, 0x0a, 0xe1, 0x75, 0x8c, 0x24,
|
||||
0x1e, 0x6e, 0xb1, 0x91, 0x1d, 0x0a, 0xf8, 0x53, 0xd4, 0xe0, 0x40, 0x45, 0x1c, 0xb5, 0x6b, 0xba,
|
||||
0xe5, 0x62, 0x5e, 0x44, 0x47, 0x89, 0xc9, 0xe2, 0x87, 0xa8, 0x19, 0x82, 0x10, 0x34, 0x80, 0x76,
|
||||
0x5d, 0x17, 0xde, 0x37, 0x85, 0xcd, 0x67, 0x59, 0x98, 0xe4, 0xf9, 0xde, 0x95, 0x85, 0xee, 0x95,
|
||||
0x73, 0x1a, 0x32, 0x21, 0xf1, 0x2f, 0x5b, 0x6e, 0x75, 0x6e, 0x76, 0x27, 0x85, 0xd6, 0x5e, 0x3d,
|
||||
0x32, 0x72, 0xad, 0x3c, 0xb2, 0xe6, 0xd4, 0x29, 0xaa, 0x33, 0x09, 0xa1, 0x9a, 0xfa, 0x5e, 0xff,
|
||||
0xe0, 0xf1, 0xe9, 0x1d, 0xb8, 0xc7, 0x7b, 0xcf, 0xe8, 0xd5, 0x07, 0x8a, 0x99, 0x64, 0x02, 0xbd,
|
||||
0x7f, 0xab, 0xeb, 0x57, 0x53, 0x0e, 0xc6, 0x09, 0x6a, 0x8a, 0xec, 0x68, 0x6e, 0x76, 0x7b, 0xf3,
|
||||
0x1a, 0x5a, 0x02, 0xe7, 0xc0, 0x21, 0xf2, 0xc1, 0x3b, 0x50, 0xf3, 0xcd, 0xa3, 0xb9, 0x0c, 0xfe,
|
||||
0x04, 0xd5, 0x03, 0x1e, 0xa7, 0x89, 0x31, 0x59, 0xd1, 0xe9, 0x53, 0x15, 0x24, 0x59, 0x4e, 0xed,
|
||||
0x6b, 0x0e, 0x5c, 0xb0, 0x38, 0xd2, 0x26, 0x5a, 0xdb, 0xd7, 0x8b, 0x2c, 0x4c, 0xf2, 0x3c, 0x1e,
|
||||
0xa1, 0x07, 0x2c, 0x12, 0xe0, 0xa7, 0x1c, 0x46, 0x33, 0x96, 0x9c, 0x0d, 0x47, 0x2f, 0x80, 0xb3,
|
||||
0xf3, 0x85, 0x76, 0x44, 0xcb, 0xfb, 0xc8, 0x00, 0x1f, 0x0c, 0x76, 0x15, 0x91, 0xdd, 0x58, 0xdc,
|
||||
0x47, 0x2d, 0x9f, 0x7a, 0x69, 0x34, 0xb9, 0xc8, 0x0c, 0x73, 0xe8, 0x1d, 0xaa, 0xed, 0x9d, 0x3e,
|
||||
0xc9, 0x62, 0xa4, 0xc8, 0xe2, 0x2f, 0x50, 0x2b, 0xe1, 0x2c, 0xe6, 0x4c, 0x2e, 0xda, 0x8d, 0x63,
|
||||
0xab, 0xbf, 0x57, 0xee, 0xfa, 0xb9, 0x89, 0x93, 0xa2, 0xa2, 0xf7, 0xa7, 0x85, 0x8e, 0x36, 0x3f,
|
||||
0x72, 0xfc, 0xca, 0x42, 0xc8, 0xcf, 0x3f, 0x2c, 0xd1, 0xb6, 0xb4, 0x0d, 0x86, 0x77, 0x60, 0x83,
|
||||
0xe2, 0x6b, 0x2d, 0xdf, 0xca, 0x22, 0x24, 0xc8, 0x9a, 0x66, 0x0f, 0xd0, 0xd1, 0xe6, 0xfa, 0xb0,
|
||||
0x8b, 0xf6, 0x23, 0x1a, 0x82, 0x48, 0xa8, 0x9f, 0x3f, 0x28, 0xef, 0x1b, 0x9a, 0xfd, 0x1f, 0xf3,
|
||||
0x04, 0x29, 0x6b, 0xf0, 0x31, 0xaa, 0xa9, 0x83, 0x59, 0x6c, 0xf1, 0x52, 0xaa, 0x5a, 0xa2, 0x33,
|
||||
0xde, 0xc3, 0xcb, 0x6b, 0xbb, 0xf2, 0xfa, 0xda, 0xae, 0x5c, 0x5d, 0xdb, 0x95, 0x57, 0x2b, 0xdb,
|
||||
0xba, 0x5c, 0xd9, 0xd6, 0xeb, 0x95, 0x6d, 0xfd, 0xbd, 0xb2, 0xad, 0x3f, 0xfe, 0xb1, 0x2b, 0x3f,
|
||||
0x37, 0xcd, 0x2d, 0xfe, 0x0b, 0x00, 0x00, 0xff, 0xff, 0xd6, 0x26, 0x9e, 0xdb, 0xcf, 0x07, 0x00,
|
||||
// 833 bytes of a gzipped FileDescriptorProto
|
||||
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x54, 0xcf, 0x4f, 0xe3, 0x46,
|
||||
0x14, 0x8e, 0x21, 0x21, 0x61, 0xa0, 0x0b, 0x9d, 0xee, 0x6a, 0x23, 0xd4, 0x1a, 0x9a, 0x4a, 0x2d,
|
||||
0x5b, 0x69, 0xed, 0xb2, 0xad, 0xfa, 0x43, 0x3d, 0x61, 0x0e, 0x08, 0x09, 0x5a, 0x34, 0x41, 0x1c,
|
||||
0xaa, 0x4a, 0xdd, 0x89, 0xf3, 0x30, 0xd3, 0xe0, 0xb1, 0x3b, 0x33, 0x8e, 0x94, 0xdb, 0xfe, 0x09,
|
||||
0xbd, 0xf4, 0x7f, 0xe2, 0xd0, 0xc3, 0x1e, 0x39, 0x45, 0x25, 0x95, 0xfa, 0x47, 0xec, 0xa9, 0x9a,
|
||||
0xf1, 0xd8, 0x0e, 0x49, 0x56, 0xbb, 0x42, 0x5c, 0xa2, 0xcc, 0x7b, 0xef, 0xfb, 0xbe, 0xf7, 0xde,
|
||||
0x7c, 0x1e, 0xf4, 0x72, 0xf0, 0xbd, 0xf4, 0x58, 0xe2, 0x0f, 0xb2, 0x1e, 0x08, 0x0e, 0x0a, 0xa4,
|
||||
0x3f, 0x04, 0xde, 0x4f, 0x84, 0x3f, 0x95, 0x78, 0x4e, 0xa3, 0x48, 0x40, 0x44, 0x55, 0x22, 0xfc,
|
||||
0x74, 0x10, 0xf9, 0x34, 0x65, 0x52, 0xff, 0x08, 0x88, 0x98, 0x54, 0x82, 0x2a, 0x96, 0x70, 0x7f,
|
||||
0xb8, 0xd7, 0x03, 0x45, 0xf7, 0xfc, 0x08, 0x38, 0x08, 0xaa, 0xa0, 0xef, 0xa5, 0x22, 0x51, 0x09,
|
||||
0xfe, 0x2e, 0x27, 0xf2, 0x34, 0xd1, 0x6f, 0x15, 0x91, 0x97, 0x0e, 0x22, 0x4f, 0x13, 0x79, 0x33,
|
||||
0x44, 0x9e, 0x25, 0xda, 0x7a, 0x1e, 0x31, 0x75, 0x99, 0xf5, 0xbc, 0x30, 0x89, 0xfd, 0x28, 0x89,
|
||||
0x12, 0xdf, 0xf0, 0xf5, 0xb2, 0x0b, 0x73, 0x32, 0x07, 0xf3, 0x2f, 0xd7, 0xd9, 0xfa, 0xc6, 0x36,
|
||||
0x4c, 0x53, 0x16, 0xd3, 0xf0, 0x92, 0x71, 0x10, 0xa3, 0xaa, 0xdb, 0x18, 0x14, 0xf5, 0x87, 0x73,
|
||||
0xdd, 0x6d, 0xf9, 0x6f, 0x43, 0x89, 0x8c, 0x2b, 0x16, 0xc3, 0x1c, 0xe0, 0xdb, 0x77, 0x01, 0x64,
|
||||
0x78, 0x09, 0x31, 0x9d, 0xc3, 0x7d, 0xfd, 0x36, 0x5c, 0xa6, 0xd8, 0x95, 0xcf, 0xb8, 0x92, 0x4a,
|
||||
0xcc, 0x82, 0x3a, 0x7f, 0x2f, 0x21, 0xb4, 0x7f, 0x7a, 0xd4, 0x05, 0x31, 0x64, 0x21, 0xe0, 0x97,
|
||||
0xa8, 0xa5, 0xe7, 0xe8, 0x53, 0x45, 0xdb, 0xce, 0x8e, 0xb3, 0xbb, 0xf6, 0xe2, 0x2b, 0xcf, 0x6e,
|
||||
0x77, 0x9a, 0xb6, 0x5a, 0xad, 0xae, 0xf6, 0x86, 0x7b, 0xde, 0xcf, 0xbd, 0xdf, 0x21, 0x54, 0x27,
|
||||
0xa0, 0x68, 0x80, 0xaf, 0xc7, 0xdb, 0xb5, 0xc9, 0x78, 0x1b, 0x55, 0x31, 0x52, 0xb2, 0x62, 0x86,
|
||||
0xea, 0x32, 0x85, 0xb0, 0xbd, 0x64, 0xd8, 0x0f, 0xbd, 0x7b, 0xde, 0x9d, 0x57, 0x35, 0xdd, 0x4d,
|
||||
0x21, 0x0c, 0xd6, 0xad, 0x68, 0x5d, 0x9f, 0x88, 0x91, 0xc0, 0x7f, 0xa0, 0x15, 0xa9, 0xa8, 0xca,
|
||||
0x64, 0x7b, 0xd9, 0x88, 0x1d, 0x3d, 0x84, 0x98, 0x21, 0x0c, 0x1e, 0x59, 0xb9, 0x95, 0xfc, 0x4c,
|
||||
0xac, 0x50, 0xe7, 0x66, 0x09, 0x7d, 0x54, 0x15, 0x1f, 0x24, 0xbc, 0xcf, 0x34, 0x11, 0xfe, 0x11,
|
||||
0xd5, 0xd5, 0x28, 0x05, 0xb3, 0xd3, 0xd5, 0xe0, 0x8b, 0xa2, 0xd9, 0xb3, 0x51, 0x0a, 0x6f, 0xc6,
|
||||
0xdb, 0x4f, 0x17, 0x40, 0x74, 0x8a, 0x18, 0x10, 0xfe, 0xa1, 0x9c, 0x63, 0xc9, 0xc0, 0x3f, 0xbd,
|
||||
0x2b, 0xfe, 0x66, 0xbc, 0xbd, 0x51, 0xc2, 0xee, 0xf6, 0x83, 0x87, 0x08, 0x5f, 0x51, 0xa9, 0xce,
|
||||
0x04, 0xe5, 0x32, 0xa7, 0x65, 0x31, 0xd8, 0x75, 0x7c, 0xf9, 0x7e, 0x37, 0xab, 0x11, 0xc1, 0x96,
|
||||
0x95, 0xc4, 0xc7, 0x73, 0x6c, 0x64, 0x81, 0x02, 0xfe, 0x1c, 0xad, 0x08, 0xa0, 0x32, 0xe1, 0xed,
|
||||
0xba, 0x69, 0xb9, 0xdc, 0x17, 0x31, 0x51, 0x62, 0xb3, 0xf8, 0x19, 0x6a, 0xc6, 0x20, 0x25, 0x8d,
|
||||
0xa0, 0xdd, 0x30, 0x85, 0x1b, 0xb6, 0xb0, 0x79, 0x92, 0x87, 0x49, 0x91, 0xef, 0xdc, 0x38, 0xe8,
|
||||
0x51, 0xb5, 0xa7, 0x63, 0x26, 0x15, 0xfe, 0x75, 0xce, 0xad, 0xde, 0xfb, 0xcd, 0xa4, 0xd1, 0xc6,
|
||||
0xab, 0x9b, 0x56, 0xae, 0x55, 0x44, 0xa6, 0x9c, 0x7a, 0x89, 0x1a, 0x4c, 0x41, 0xac, 0xb7, 0xbe,
|
||||
0xbc, 0xbb, 0xf6, 0xe2, 0xe0, 0x01, 0xdc, 0x13, 0x7c, 0x60, 0xf5, 0x1a, 0x47, 0x9a, 0x99, 0xe4,
|
||||
0x02, 0x9d, 0xff, 0x96, 0xa7, 0x47, 0xd3, 0x0e, 0xc6, 0x29, 0x6a, 0xca, 0xfc, 0x68, 0x27, 0xbb,
|
||||
0xbf, 0x79, 0x2d, 0x2d, 0x81, 0x0b, 0x10, 0xc0, 0x43, 0x08, 0xd6, 0xf4, 0x7e, 0x8b, 0x68, 0x21,
|
||||
0x83, 0x3f, 0x43, 0x8d, 0x48, 0x24, 0x59, 0x6a, 0x4d, 0x56, 0x76, 0x7a, 0xa8, 0x83, 0x24, 0xcf,
|
||||
0xe9, 0xfb, 0x1a, 0x82, 0x90, 0x2c, 0xe1, 0xc6, 0x44, 0x53, 0xf7, 0x75, 0x9e, 0x87, 0x49, 0x91,
|
||||
0xc7, 0x5d, 0xf4, 0x84, 0x71, 0x09, 0x61, 0x26, 0xa0, 0x3b, 0x60, 0xe9, 0xd9, 0x71, 0xf7, 0x1c,
|
||||
0x04, 0xbb, 0x18, 0x19, 0x47, 0xb4, 0x82, 0x4f, 0x2c, 0xf0, 0xc9, 0xd1, 0xa2, 0x22, 0xb2, 0x18,
|
||||
0x8b, 0x77, 0x51, 0x2b, 0xa4, 0x41, 0xc6, 0xfb, 0x57, 0xb9, 0x61, 0xd6, 0x83, 0x75, 0x7d, 0x7b,
|
||||
0x07, 0xfb, 0x79, 0x8c, 0x94, 0x59, 0x7c, 0x8a, 0x1e, 0x9b, 0x96, 0x4f, 0x05, 0x4b, 0x04, 0x53,
|
||||
0xa3, 0x13, 0xc6, 0x59, 0x9c, 0xc5, 0xed, 0xe6, 0x8e, 0xb3, 0xdb, 0x08, 0x3e, 0xb6, 0xea, 0x8f,
|
||||
0x0f, 0x17, 0xd4, 0x90, 0x85, 0x48, 0xbc, 0x8f, 0x36, 0xec, 0x6c, 0x45, 0xa6, 0xdd, 0x32, 0x64,
|
||||
0x4f, 0x2d, 0xd9, 0xc6, 0xf9, 0xdd, 0x34, 0x99, 0xad, 0xef, 0xfc, 0xe5, 0xa0, 0xcd, 0xd9, 0xb7,
|
||||
0x04, 0xbf, 0x72, 0x10, 0x0a, 0x8b, 0xef, 0x57, 0xb6, 0x1d, 0xe3, 0xb6, 0xe3, 0x07, 0x70, 0x5b,
|
||||
0xf9, 0x28, 0x54, 0x4f, 0x72, 0x19, 0x92, 0x64, 0x4a, 0xb3, 0x03, 0x68, 0x73, 0xd6, 0x25, 0xd8,
|
||||
0x47, 0xab, 0x9c, 0xc6, 0x20, 0x53, 0x1a, 0x16, 0xef, 0xd6, 0x87, 0x96, 0x66, 0xf5, 0xa7, 0x22,
|
||||
0x41, 0xaa, 0x1a, 0xbc, 0x83, 0xea, 0xfa, 0x60, 0xfd, 0x53, 0x3e, 0xc8, 0xba, 0x96, 0x98, 0x4c,
|
||||
0xf0, 0xec, 0xfa, 0xd6, 0xad, 0xbd, 0xbe, 0x75, 0x6b, 0x37, 0xb7, 0x6e, 0xed, 0xd5, 0xc4, 0x75,
|
||||
0xae, 0x27, 0xae, 0xf3, 0x7a, 0xe2, 0x3a, 0xff, 0x4c, 0x5c, 0xe7, 0xcf, 0x7f, 0xdd, 0xda, 0x2f,
|
||||
0x4d, 0x3b, 0xc5, 0xff, 0x01, 0x00, 0x00, 0xff, 0xff, 0x1f, 0x2d, 0x18, 0xd6, 0x36, 0x08, 0x00,
|
||||
0x00,
|
||||
}
|
||||
|
||||
@ -91,14 +91,19 @@ message APIServiceSpec {
|
||||
// CABundle is a PEM encoded CA bundle which will be used to validate an API server's serving certificate.
|
||||
optional bytes caBundle = 5;
|
||||
|
||||
// Priority controls the ordering of this API group in the overall discovery document that gets served.
|
||||
// Client tools like `kubectl` use this ordering to derive preference, so this ordering mechanism is important.
|
||||
// Values must be between 1 and 1000
|
||||
// The primary sort is based on priority, ordered lowest number to highest (10 before 20).
|
||||
// GroupPriorityMininum is the priority this group should have at least. Higher priority means that the group is prefered by clients over lower priority ones.
|
||||
// Note that other versions of this group might specify even higher GroupPriorityMininum values such that the whole group gets a higher priority.
|
||||
// The primary sort is based on GroupPriorityMinimum, ordered highest number to lowest (20 before 10).
|
||||
// The secondary sort is based on the alphabetical comparison of the name of the object. (v1.bar before v1.foo)
|
||||
// We'd recommend something like: *.k8s.io (except extensions) at 100, extensions at 150
|
||||
// PaaSes (OpenShift, Deis) are recommended to be in the 200s
|
||||
optional int64 priority = 6;
|
||||
// We'd recommend something like: *.k8s.io (except extensions) at 18000 and
|
||||
// PaaSes (OpenShift, Deis) are recommended to be in the 2000s
|
||||
optional int32 groupPriorityMinimum = 7;
|
||||
|
||||
// VersionPriority controls the ordering of this API version inside of its group. Must be greater than zero.
|
||||
// The primary sort is based on VersionPriority, ordered highest to lowest (20 before 10).
|
||||
// The secondary sort is based on the alphabetical comparison of the name of the object. (v1.bar before v1.foo)
|
||||
// Since it's inside of a group, the number can be small, probably in the 10s.
|
||||
optional int32 versionPriority = 8;
|
||||
}
|
||||
|
||||
// APIServiceStatus contains derived information about an API server
|
||||
|
||||
@ -53,14 +53,22 @@ type APIServiceSpec struct {
|
||||
// CABundle is a PEM encoded CA bundle which will be used to validate an API server's serving certificate.
|
||||
CABundle []byte `json:"caBundle" protobuf:"bytes,5,opt,name=caBundle"`
|
||||
|
||||
// Priority controls the ordering of this API group in the overall discovery document that gets served.
|
||||
// Client tools like `kubectl` use this ordering to derive preference, so this ordering mechanism is important.
|
||||
// Values must be between 1 and 1000
|
||||
// The primary sort is based on priority, ordered lowest number to highest (10 before 20).
|
||||
// GroupPriorityMininum is the priority this group should have at least. Higher priority means that the group is prefered by clients over lower priority ones.
|
||||
// Note that other versions of this group might specify even higher GroupPriorityMininum values such that the whole group gets a higher priority.
|
||||
// The primary sort is based on GroupPriorityMinimum, ordered highest number to lowest (20 before 10).
|
||||
// The secondary sort is based on the alphabetical comparison of the name of the object. (v1.bar before v1.foo)
|
||||
// We'd recommend something like: *.k8s.io (except extensions) at 100, extensions at 150
|
||||
// PaaSes (OpenShift, Deis) are recommended to be in the 200s
|
||||
Priority int64 `json:"priority" protobuf:"varint,6,opt,name=priority"`
|
||||
// We'd recommend something like: *.k8s.io (except extensions) at 18000 and
|
||||
// PaaSes (OpenShift, Deis) are recommended to be in the 2000s
|
||||
GroupPriorityMinimum int32 `json:"groupPriorityMinimum" protobuf:"varint,7,opt,name=groupPriorityMinimum"`
|
||||
|
||||
// VersionPriority controls the ordering of this API version inside of its group. Must be greater than zero.
|
||||
// The primary sort is based on VersionPriority, ordered highest to lowest (20 before 10).
|
||||
// The secondary sort is based on the alphabetical comparison of the name of the object. (v1.bar before v1.foo)
|
||||
// Since it's inside of a group, the number can be small, probably in the 10s.
|
||||
VersionPriority int32 `json:"versionPriority" protobuf:"varint,8,opt,name=versionPriority"`
|
||||
|
||||
// leaving this here so everyone remembers why proto index 6 is skipped
|
||||
// Priority int64 `json:"priority" protobuf:"varint,6,opt,name=priority"`
|
||||
}
|
||||
|
||||
type ConditionStatus string
|
||||
|
||||
@ -142,7 +142,8 @@ func autoConvert_v1beta1_APIServiceSpec_To_apiregistration_APIServiceSpec(in *AP
|
||||
out.Version = in.Version
|
||||
out.InsecureSkipTLSVerify = in.InsecureSkipTLSVerify
|
||||
out.CABundle = *(*[]byte)(unsafe.Pointer(&in.CABundle))
|
||||
out.Priority = in.Priority
|
||||
out.GroupPriorityMinimum = in.GroupPriorityMinimum
|
||||
out.VersionPriority = in.VersionPriority
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -161,7 +162,8 @@ func autoConvert_apiregistration_APIServiceSpec_To_v1beta1_APIServiceSpec(in *ap
|
||||
} else {
|
||||
out.CABundle = *(*[]byte)(unsafe.Pointer(&in.CABundle))
|
||||
}
|
||||
out.Priority = in.Priority
|
||||
out.GroupPriorityMinimum = in.GroupPriorityMinimum
|
||||
out.VersionPriority = in.VersionPriority
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@ -58,9 +58,11 @@ func ValidateAPIService(apiService *apiregistration.APIService) field.ErrorList
|
||||
allErrs = append(allErrs, field.Invalid(field.NewPath("spec", "version"), apiService.Spec.Version, errString))
|
||||
}
|
||||
|
||||
if apiService.Spec.Priority <= 0 || apiService.Spec.Priority > 1000 {
|
||||
allErrs = append(allErrs, field.Invalid(field.NewPath("spec", "priority"), apiService.Spec.Priority, "priority must be positive and less than 1000"))
|
||||
|
||||
if apiService.Spec.GroupPriorityMinimum <= 0 || apiService.Spec.GroupPriorityMinimum > 20000 {
|
||||
allErrs = append(allErrs, field.Invalid(field.NewPath("spec", "groupPriorityMinimum"), apiService.Spec.GroupPriorityMinimum, "must be positive and less than 20000"))
|
||||
}
|
||||
if apiService.Spec.VersionPriority <= 0 || apiService.Spec.VersionPriority > 1000 {
|
||||
allErrs = append(allErrs, field.Invalid(field.NewPath("spec", "versionPriority"), apiService.Spec.VersionPriority, "must be positive and less than 1000"))
|
||||
}
|
||||
|
||||
if apiService.Spec.Service == nil {
|
||||
|
||||
@ -73,7 +73,7 @@ func (r *apisHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
apiServicesByGroup := apiregistrationapi.SortedByGroup(apiServices)
|
||||
apiServicesByGroup := apiregistrationapi.SortedByGroupAndVersion(apiServices)
|
||||
for _, apiGroupServers := range apiServicesByGroup {
|
||||
// skip the legacy group
|
||||
if len(apiGroupServers[0].Spec.Group) == 0 {
|
||||
@ -91,7 +91,7 @@ func (r *apisHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
|
||||
// convertToDiscoveryAPIGroup takes apiservices in a single group and returns a discovery compatible object.
|
||||
// if none of the services are available, it will return nil.
|
||||
func convertToDiscoveryAPIGroup(apiServices []*apiregistrationapi.APIService) *metav1.APIGroup {
|
||||
apiServicesByGroup := apiregistrationapi.SortedByGroup(apiServices)[0]
|
||||
apiServicesByGroup := apiregistrationapi.SortedByGroupAndVersion(apiServices)[0]
|
||||
|
||||
var discoveryGroup *metav1.APIGroup
|
||||
|
||||
|
||||
@ -60,9 +60,9 @@ func TestAPIs(t *testing.T) {
|
||||
Namespace: "ns",
|
||||
Name: "api",
|
||||
},
|
||||
Group: "foo",
|
||||
Version: "v1",
|
||||
Priority: 10,
|
||||
Group: "foo",
|
||||
Version: "v1",
|
||||
GroupPriorityMinimum: 11,
|
||||
},
|
||||
Status: apiregistration.APIServiceStatus{
|
||||
Conditions: []apiregistration.APIServiceCondition{
|
||||
@ -77,9 +77,9 @@ func TestAPIs(t *testing.T) {
|
||||
Namespace: "ns",
|
||||
Name: "api",
|
||||
},
|
||||
Group: "bar",
|
||||
Version: "v1",
|
||||
Priority: 11,
|
||||
Group: "bar",
|
||||
Version: "v1",
|
||||
GroupPriorityMinimum: 10,
|
||||
},
|
||||
Status: apiregistration.APIServiceStatus{
|
||||
Conditions: []apiregistration.APIServiceCondition{
|
||||
@ -131,9 +131,10 @@ func TestAPIs(t *testing.T) {
|
||||
Namespace: "ns",
|
||||
Name: "api",
|
||||
},
|
||||
Group: "foo",
|
||||
Version: "v1",
|
||||
Priority: 20,
|
||||
Group: "foo",
|
||||
Version: "v1",
|
||||
GroupPriorityMinimum: 20,
|
||||
VersionPriority: 10,
|
||||
},
|
||||
Status: apiregistration.APIServiceStatus{
|
||||
Conditions: []apiregistration.APIServiceCondition{
|
||||
@ -148,9 +149,9 @@ func TestAPIs(t *testing.T) {
|
||||
Namespace: "ns",
|
||||
Name: "api",
|
||||
},
|
||||
Group: "bar",
|
||||
Version: "v2",
|
||||
Priority: 11,
|
||||
Group: "bar",
|
||||
Version: "v2",
|
||||
GroupPriorityMinimum: 11,
|
||||
},
|
||||
Status: apiregistration.APIServiceStatus{
|
||||
Conditions: []apiregistration.APIServiceCondition{
|
||||
@ -165,9 +166,10 @@ func TestAPIs(t *testing.T) {
|
||||
Namespace: "ns",
|
||||
Name: "api",
|
||||
},
|
||||
Group: "foo",
|
||||
Version: "v2",
|
||||
Priority: 1,
|
||||
Group: "foo",
|
||||
Version: "v2",
|
||||
GroupPriorityMinimum: 1,
|
||||
VersionPriority: 15,
|
||||
},
|
||||
Status: apiregistration.APIServiceStatus{
|
||||
Conditions: []apiregistration.APIServiceCondition{
|
||||
@ -182,9 +184,9 @@ func TestAPIs(t *testing.T) {
|
||||
Namespace: "ns",
|
||||
Name: "api",
|
||||
},
|
||||
Group: "bar",
|
||||
Version: "v1",
|
||||
Priority: 11,
|
||||
Group: "bar",
|
||||
Version: "v1",
|
||||
GroupPriorityMinimum: 11,
|
||||
},
|
||||
Status: apiregistration.APIServiceStatus{
|
||||
Conditions: []apiregistration.APIServiceCondition{
|
||||
@ -336,9 +338,10 @@ func TestAPIGroup(t *testing.T) {
|
||||
Namespace: "ns",
|
||||
Name: "api",
|
||||
},
|
||||
Group: "foo",
|
||||
Version: "v1",
|
||||
Priority: 20,
|
||||
Group: "foo",
|
||||
Version: "v1",
|
||||
GroupPriorityMinimum: 20,
|
||||
VersionPriority: 10,
|
||||
},
|
||||
Status: apiregistration.APIServiceStatus{
|
||||
Conditions: []apiregistration.APIServiceCondition{
|
||||
@ -353,9 +356,9 @@ func TestAPIGroup(t *testing.T) {
|
||||
Namespace: "ns",
|
||||
Name: "api",
|
||||
},
|
||||
Group: "bar",
|
||||
Version: "v2",
|
||||
Priority: 11,
|
||||
Group: "bar",
|
||||
Version: "v2",
|
||||
GroupPriorityMinimum: 11,
|
||||
},
|
||||
Status: apiregistration.APIServiceStatus{
|
||||
Conditions: []apiregistration.APIServiceCondition{
|
||||
@ -370,9 +373,10 @@ func TestAPIGroup(t *testing.T) {
|
||||
Namespace: "ns",
|
||||
Name: "api",
|
||||
},
|
||||
Group: "foo",
|
||||
Version: "v2",
|
||||
Priority: 1,
|
||||
Group: "foo",
|
||||
Version: "v2",
|
||||
GroupPriorityMinimum: 1,
|
||||
VersionPriority: 15,
|
||||
},
|
||||
Status: apiregistration.APIServiceStatus{
|
||||
Conditions: []apiregistration.APIServiceCondition{
|
||||
@ -387,9 +391,9 @@ func TestAPIGroup(t *testing.T) {
|
||||
Namespace: "ns",
|
||||
Name: "api",
|
||||
},
|
||||
Group: "bar",
|
||||
Version: "v1",
|
||||
Priority: 11,
|
||||
Group: "bar",
|
||||
Version: "v1",
|
||||
GroupPriorityMinimum: 11,
|
||||
},
|
||||
Status: apiregistration.APIServiceStatus{
|
||||
Conditions: []apiregistration.APIServiceCondition{
|
||||
|
||||
@ -315,10 +315,11 @@ func TestAggregatedAPIServer(t *testing.T) {
|
||||
Namespace: "kube-wardle",
|
||||
Name: "api",
|
||||
},
|
||||
Group: "wardle.k8s.io",
|
||||
Version: "v1alpha1",
|
||||
CABundle: wardleCA,
|
||||
Priority: 200,
|
||||
Group: "wardle.k8s.io",
|
||||
Version: "v1alpha1",
|
||||
CABundle: wardleCA,
|
||||
GroupPriorityMinimum: 200,
|
||||
VersionPriority: 200,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
@ -337,9 +338,10 @@ func TestAggregatedAPIServer(t *testing.T) {
|
||||
Spec: apiregistrationv1beta1.APIServiceSpec{
|
||||
// register this as a loca service so it doesn't try to lookup the default kubernetes service
|
||||
// which will have an unroutable IP address since its fake.
|
||||
Group: "",
|
||||
Version: "v1",
|
||||
Priority: 100,
|
||||
Group: "",
|
||||
Version: "v1",
|
||||
GroupPriorityMinimum: 100,
|
||||
VersionPriority: 100,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
|
||||
Loading…
Reference in New Issue
Block a user