diff --git a/pkg/controlplane/instance.go b/pkg/controlplane/instance.go index 6a3e3c02f40..5456849e42a 100644 --- a/pkg/controlplane/instance.go +++ b/pkg/controlplane/instance.go @@ -55,6 +55,7 @@ import ( storageapiv1alpha1 "k8s.io/api/storage/v1alpha1" storageapiv1beta1 "k8s.io/api/storage/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" utilnet "k8s.io/apimachinery/pkg/util/net" "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/wait" @@ -571,7 +572,7 @@ func (m *Instance) InstallAPIs(apiResourceConfigSource serverstorage.APIResource if len(apiGroupInfo.VersionedResourcesStorageMap) == 0 { // If we have no storage for any resource configured, this API group is effectively disabled. // This can happen when an entire API group, version, or development-stage (alpha, beta, GA) is disabled. - klog.Warningf("API group %q is not enabled, skipping.", groupName) + klog.Infof("API group %q is not enabled, skipping.", groupName) continue } @@ -638,11 +639,9 @@ func (n nodeAddressProvider) externalAddresses() ([]string, error) { return addrs, nil } -// DefaultAPIResourceConfigSource returns default configuration for an APIResource. -func DefaultAPIResourceConfigSource() *serverstorage.ResourceConfig { - ret := serverstorage.NewResourceConfig() - // NOTE: GroupVersions listed here will be enabled by default. Don't put alpha versions in the list. - ret.EnableVersions( +var ( + // stableAPIGroupVersionsEnabledByDefault is a list of our stable versions. + stableAPIGroupVersionsEnabledByDefault = []schema.GroupVersion{ admissionregistrationv1.SchemeGroupVersion, apiv1.SchemeGroupVersion, appsv1.SchemeGroupVersion, @@ -650,34 +649,73 @@ func DefaultAPIResourceConfigSource() *serverstorage.ResourceConfig { authorizationapiv1.SchemeGroupVersion, autoscalingapiv1.SchemeGroupVersion, autoscalingapiv2.SchemeGroupVersion, - autoscalingapiv2beta1.SchemeGroupVersion, - autoscalingapiv2beta2.SchemeGroupVersion, batchapiv1.SchemeGroupVersion, - batchapiv1beta1.SchemeGroupVersion, certificatesapiv1.SchemeGroupVersion, coordinationapiv1.SchemeGroupVersion, discoveryv1.SchemeGroupVersion, - discoveryv1beta1.SchemeGroupVersion, eventsv1.SchemeGroupVersion, - eventsv1beta1.SchemeGroupVersion, networkingapiv1.SchemeGroupVersion, nodev1.SchemeGroupVersion, - nodev1beta1.SchemeGroupVersion, // remove in 1.26 policyapiv1.SchemeGroupVersion, - policyapiv1beta1.SchemeGroupVersion, rbacv1.SchemeGroupVersion, storageapiv1.SchemeGroupVersion, - storageapiv1beta1.SchemeGroupVersion, schedulingapiv1.SchemeGroupVersion, - flowcontrolv1beta2.SchemeGroupVersion, + } + + // legacyBetaEnabledByDefaultResources is the list of beta resources we enable. You may only add to this list + // if your resource is already enabled by default in a beta level we still serve AND there is no stable API for it. + // see https://github.com/kubernetes/enhancements/tree/master/keps/sig-architecture/3136-beta-apis-off-by-default + // for more details. + legacyBetaEnabledByDefaultResources = []schema.GroupVersionResource{ + autoscalingapiv2beta1.SchemeGroupVersion.WithResource("horizontalpodautoscalers"), // remove in 1.25 + autoscalingapiv2beta2.SchemeGroupVersion.WithResource("horizontalpodautoscalers"), // remove in 1.26 + batchapiv1beta1.SchemeGroupVersion.WithResource("cronjobs"), // remove in 1.25 + discoveryv1beta1.SchemeGroupVersion.WithResource("endpointslices"), // remove in 1.25 + eventsv1beta1.SchemeGroupVersion.WithResource("events"), // remove in 1.25 + nodev1beta1.SchemeGroupVersion.WithResource("runtimeclasses"), // remove in 1.25 + policyapiv1beta1.SchemeGroupVersion.WithResource("poddisruptionbudgets"), // remove in 1.25 + policyapiv1beta1.SchemeGroupVersion.WithResource("podsecuritypolicies"), // remove in 1.25 + storageapiv1beta1.SchemeGroupVersion.WithResource("csinodes"), // remove in 1.25 + storageapiv1beta1.SchemeGroupVersion.WithResource("csistoragecapacities"), // remove in 1.27 + flowcontrolv1beta1.SchemeGroupVersion.WithResource("flowschemas"), // remove in 1.26 + flowcontrolv1beta1.SchemeGroupVersion.WithResource("prioritylevelconfigurations"), // remove in 1.26 + flowcontrolv1beta2.SchemeGroupVersion.WithResource("flowschemas"), // remove in 1.29 + flowcontrolv1beta2.SchemeGroupVersion.WithResource("prioritylevelconfigurations"), // remove in 1.29 + } + // betaAPIGroupVersionsDisabledByDefault is for all future beta groupVersions. + betaAPIGroupVersionsDisabledByDefault = []schema.GroupVersion{ + autoscalingapiv2beta1.SchemeGroupVersion, + autoscalingapiv2beta2.SchemeGroupVersion, + batchapiv1beta1.SchemeGroupVersion, + discoveryv1beta1.SchemeGroupVersion, + eventsv1beta1.SchemeGroupVersion, + nodev1beta1.SchemeGroupVersion, // remove in 1.26 + policyapiv1beta1.SchemeGroupVersion, + storageapiv1beta1.SchemeGroupVersion, flowcontrolv1beta1.SchemeGroupVersion, - ) - // disable alpha versions explicitly so we have a full list of what's possible to serve - ret.DisableVersions( + flowcontrolv1beta2.SchemeGroupVersion, + } + + // alphaAPIGroupVersionsDisabledByDefault holds the alpha APIs we have. They are always disabled by default. + alphaAPIGroupVersionsDisabledByDefault = []schema.GroupVersion{ apiserverinternalv1alpha1.SchemeGroupVersion, storageapiv1alpha1.SchemeGroupVersion, flowcontrolv1alpha1.SchemeGroupVersion, - ) + } +) + +// DefaultAPIResourceConfigSource returns default configuration for an APIResource. +func DefaultAPIResourceConfigSource() *serverstorage.ResourceConfig { + ret := serverstorage.NewResourceConfig() + // NOTE: GroupVersions listed here will be enabled by default. Don't put alpha or beta versions in the list. + ret.EnableVersions(stableAPIGroupVersionsEnabledByDefault...) + + // disable alpha and beta versions explicitly so we have a full list of what's possible to serve + ret.DisableVersions(betaAPIGroupVersionsDisabledByDefault...) + ret.DisableVersions(alphaAPIGroupVersionsDisabledByDefault...) + + // enable the legacy beta resources that were present before stopped serving new beta APIs by default. + ret.EnableResources(legacyBetaEnabledByDefaultResources...) return ret } diff --git a/pkg/controlplane/instance_test.go b/pkg/controlplane/instance_test.go index 4f4471d42bf..ff5e56f1b28 100644 --- a/pkg/controlplane/instance_test.go +++ b/pkg/controlplane/instance_test.go @@ -28,8 +28,16 @@ import ( "strings" "testing" + autoscalingapiv2beta1 "k8s.io/api/autoscaling/v2beta1" + autoscalingapiv2beta2 "k8s.io/api/autoscaling/v2beta2" + batchapiv1beta1 "k8s.io/api/batch/v1beta1" certificatesapiv1beta1 "k8s.io/api/certificates/v1beta1" apiv1 "k8s.io/api/core/v1" + discoveryv1beta1 "k8s.io/api/discovery/v1beta1" + eventsv1beta1 "k8s.io/api/events/v1beta1" + nodev1beta1 "k8s.io/api/node/v1beta1" + policyapiv1beta1 "k8s.io/api/policy/v1beta1" + storageapiv1beta1 "k8s.io/api/storage/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" utilnet "k8s.io/apimachinery/pkg/util/net" @@ -50,6 +58,7 @@ import ( kubeversion "k8s.io/component-base/version" "k8s.io/kubernetes/pkg/api/legacyscheme" "k8s.io/kubernetes/pkg/apis/batch" + flowcontrolv1beta2 "k8s.io/kubernetes/pkg/apis/flowcontrol/v1beta2" "k8s.io/kubernetes/pkg/apis/networking" apisstorage "k8s.io/kubernetes/pkg/apis/storage" "k8s.io/kubernetes/pkg/controlplane/reconcilers" @@ -460,3 +469,56 @@ func TestNoAlphaVersionsEnabledByDefault(t *testing.T) { } } } + +func TestNoBetaVersionsEnabledByDefault(t *testing.T) { + config := DefaultAPIResourceConfigSource() + for gv, enable := range config.GroupVersionConfigs { + if enable && strings.Contains(gv.Version, "beta") { + t.Errorf("Beta API version %s enabled by default", gv.String()) + } + } +} + +func TestNewBetaResourcesEnabledByDefault(t *testing.T) { + // legacyEnabledBetaResources is nearly a duplication from elsewhere. This is intentional. These types already have + // GA equivalents available and should therefore never have a beta enabled by default again. + legacyEnabledBetaResources := map[schema.GroupVersionResource]bool{ + autoscalingapiv2beta1.SchemeGroupVersion.WithResource("horizontalpodautoscalers"): true, + autoscalingapiv2beta2.SchemeGroupVersion.WithResource("horizontalpodautoscalers"): true, + batchapiv1beta1.SchemeGroupVersion.WithResource("cronjobs"): true, + discoveryv1beta1.SchemeGroupVersion.WithResource("endpointslices"): true, + eventsv1beta1.SchemeGroupVersion.WithResource("events"): true, + nodev1beta1.SchemeGroupVersion.WithResource("runtimeclasses"): true, + policyapiv1beta1.SchemeGroupVersion.WithResource("poddisruptionbudgets"): true, + policyapiv1beta1.SchemeGroupVersion.WithResource("podsecuritypolicies"): true, + storageapiv1beta1.SchemeGroupVersion.WithResource("csinodes"): true, + storageapiv1beta1.SchemeGroupVersion.WithResource("csistoragecapacities"): true, + } + + // legacyBetaResourcesWithoutStableEquivalents contains those groupresources that were enabled by default as beta + // before we changed that policy and do not have stable versions. These resources are allowed to have additional + // beta versions enabled by default. Nothing new should be added here. There are no future exceptions because there + // are no more beta resources enabled by default. + legacyBetaResourcesWithoutStableEquivalents := map[schema.GroupResource]bool{ + storageapiv1beta1.SchemeGroupVersion.WithResource("csistoragecapacities").GroupResource(): true, + flowcontrolv1beta2.SchemeGroupVersion.WithResource("flowschemas").GroupResource(): true, + flowcontrolv1beta2.SchemeGroupVersion.WithResource("prioritylevelconfigurations").GroupResource(): true, + } + + config := DefaultAPIResourceConfigSource() + for gvr, enable := range config.ResourceConfigs { + if !strings.Contains(gvr.Version, "beta") { + continue // only check beta things + } + if !enable { + continue // only check things that are enabled + } + if legacyEnabledBetaResources[gvr] { + continue // this is a legacy beta resource + } + if legacyBetaResourcesWithoutStableEquivalents[gvr.GroupResource()] { + continue // this is another beta of a legacy beta resource with no stable equivalent + } + t.Errorf("no new beta resources can be enabled by default, see https://github.com/kubernetes/enhancements/blob/0ad0fc8269165ca300d05ca51c7ce190a79976a5/keps/sig-architecture/3136-beta-apis-off-by-default/README.md: %v", gvr) + } +} diff --git a/staging/src/k8s.io/apiserver/pkg/server/storage/storage_factory.go b/staging/src/k8s.io/apiserver/pkg/server/storage/storage_factory.go index 0ba0d75f8b7..d8de4cd84fb 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/storage/storage_factory.go +++ b/staging/src/k8s.io/apiserver/pkg/server/storage/storage_factory.go @@ -240,7 +240,7 @@ func getAllResourcesAlias(resource schema.GroupResource) schema.GroupResource { func (s *DefaultStorageFactory) getStorageGroupResource(groupResource schema.GroupResource) schema.GroupResource { for _, potentialStorageResource := range s.Overrides[groupResource].cohabitatingResources { - // TODO determine if have ever stored any of our cohabitating resources in a different location on new clusters + // TODO deads2k or liggitt determine if have ever stored any of our cohabitating resources in a different location on new clusters if s.APIResourceConfigSource.AnyResourceForGroupEnabled(potentialStorageResource.Group) { return potentialStorageResource }