remove more CRD v1beta1 client dependencies

This commit is contained in:
David Eads 2021-03-04 13:55:31 -05:00
parent aeb6508e1e
commit c14ff1a674
12 changed files with 321 additions and 204 deletions

View File

@ -0,0 +1,55 @@
/*
Copyright 2021 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package fixtures
import (
"context"
"encoding/json"
"path"
"go.etcd.io/etcd/clientv3"
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
"k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
genericapirequest "k8s.io/apiserver/pkg/endpoints/request"
"k8s.io/client-go/dynamic"
)
// CreateCRDUsingRemovedAPI creates a CRD directly using etcd. This is must *ONLY* be used for checks of compatibility
// with removed data. Do not use this just because you don't want to update your test to use v1. Only use this
// when it actually matters.
func CreateCRDUsingRemovedAPI(etcdClient *clientv3.Client, etcdStoragePrefix string, betaCRD *apiextensionsv1beta1.CustomResourceDefinition, apiExtensionsClient clientset.Interface, dynamicClientSet dynamic.Interface) (*apiextensionsv1.CustomResourceDefinition, error) {
// attempt defaulting, best effort
apiextensionsv1beta1.SetDefaults_CustomResourceDefinition(betaCRD)
betaCRD.Kind = "CustomResourceDefinition"
betaCRD.APIVersion = apiextensionsv1beta1.SchemeGroupVersion.Group + "/" + apiextensionsv1beta1.SchemeGroupVersion.Version
ctx := genericapirequest.WithNamespace(genericapirequest.NewContext(), metav1.NamespaceNone)
key := path.Join("/", etcdStoragePrefix, "apiextensions.k8s.io", "customresourcedefinitions", betaCRD.Name)
val, _ := json.Marshal(betaCRD)
if _, err := etcdClient.Put(ctx, key, string(val)); err != nil {
return nil, err
}
crd, err := apiExtensionsClient.ApiextensionsV1().CustomResourceDefinitions().Get(context.TODO(), betaCRD.Name, metav1.GetOptions{})
if err != nil {
return nil, err
}
return waitForCRDReady(crd, apiExtensionsClient, dynamicClientSet)
}

View File

@ -24,7 +24,6 @@ import (
"k8s.io/utils/pointer"
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
"k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@ -246,19 +245,6 @@ func NewCurletInstance(namespace, name string) *unstructured.Unstructured {
}
}
func servedVersions(crd *apiextensionsv1beta1.CustomResourceDefinition) []string {
if len(crd.Spec.Versions) == 0 {
return []string{crd.Spec.Version}
}
var versions []string
for _, v := range crd.Spec.Versions {
if v.Served {
versions = append(versions, v.Name)
}
}
return versions
}
func servedV1Versions(crd *apiextensionsv1.CustomResourceDefinition) []string {
if len(crd.Spec.Versions) == 0 {
return []string{}
@ -272,22 +258,6 @@ func servedV1Versions(crd *apiextensionsv1.CustomResourceDefinition) []string {
return versions
}
func existsInDiscovery(crd *apiextensionsv1beta1.CustomResourceDefinition, apiExtensionsClient clientset.Interface, version string) (bool, error) {
groupResource, err := apiExtensionsClient.Discovery().ServerResourcesForGroupVersion(crd.Spec.Group + "/" + version)
if err != nil {
if errors.IsNotFound(err) {
return false, nil
}
return false, err
}
for _, g := range groupResource.APIResources {
if g.Name == crd.Spec.Names.Plural {
return true, nil
}
}
return false, nil
}
func existsInDiscoveryV1(crd *apiextensionsv1.CustomResourceDefinition, apiExtensionsClient clientset.Interface, version string) (bool, error) {
groupResource, err := apiExtensionsClient.Discovery().ServerResourcesForGroupVersion(crd.Spec.Group + "/" + version)
if err != nil {
@ -304,37 +274,27 @@ func existsInDiscoveryV1(crd *apiextensionsv1.CustomResourceDefinition, apiExten
return false, nil
}
// CreateNewCustomResourceDefinitionWatchUnsafe creates the CRD and makes sure
// waitForCRDReadyWatchUnsafe creates the CRD and makes sure
// the apiextension apiserver has installed the CRD. But it's not safe to watch
// the created CR. Please call CreateNewCustomResourceDefinition if you need to
// the created CR. Please call CreateCRDUsingRemovedAPI if you need to
// watch the CR.
func CreateNewCustomResourceDefinitionWatchUnsafe(crd *apiextensionsv1beta1.CustomResourceDefinition, apiExtensionsClient clientset.Interface) (*apiextensionsv1beta1.CustomResourceDefinition, error) {
crd, err := apiExtensionsClient.ApiextensionsV1beta1().CustomResourceDefinitions().Create(context.TODO(), crd, metav1.CreateOptions{})
if err != nil {
return nil, err
}
func waitForCRDReadyWatchUnsafe(crd *apiextensionsv1.CustomResourceDefinition, apiExtensionsClient clientset.Interface) (*apiextensionsv1.CustomResourceDefinition, error) {
// wait until all resources appears in discovery
for _, version := range servedVersions(crd) {
for _, version := range servedV1Versions(crd) {
err := wait.PollImmediate(500*time.Millisecond, 30*time.Second, func() (bool, error) {
return existsInDiscovery(crd, apiExtensionsClient, version)
return existsInDiscoveryV1(crd, apiExtensionsClient, version)
})
if err != nil {
return nil, err
}
}
return crd, err
return crd, nil
}
// CreateNewCustomResourceDefinition creates the given CRD and makes sure its watch cache is primed on the server.
func CreateNewCustomResourceDefinition(crd *apiextensionsv1beta1.CustomResourceDefinition, apiExtensionsClient clientset.Interface, dynamicClientSet dynamic.Interface) (*apiextensionsv1beta1.CustomResourceDefinition, error) {
crd, err := CreateNewCustomResourceDefinitionWatchUnsafe(crd, apiExtensionsClient)
if err != nil {
return nil, err
}
v1CRD, err := apiExtensionsClient.ApiextensionsV1().CustomResourceDefinitions().Get(context.TODO(), crd.Name, metav1.GetOptions{})
// waitForCRDReady creates the given CRD and makes sure its watch cache is primed on the server.
func waitForCRDReady(crd *apiextensionsv1.CustomResourceDefinition, apiExtensionsClient clientset.Interface, dynamicClientSet dynamic.Interface) (*apiextensionsv1.CustomResourceDefinition, error) {
v1CRD, err := waitForCRDReadyWatchUnsafe(crd, apiExtensionsClient)
if err != nil {
return nil, err
}
@ -355,7 +315,7 @@ func CreateNewCustomResourceDefinition(crd *apiextensionsv1beta1.CustomResourceD
if err != nil {
return nil, err
}
return crd, nil
return v1CRD, err
}
// CreateNewV1CustomResourceDefinitionWatchUnsafe creates the CRD and makes sure
@ -419,7 +379,7 @@ func resourceClientForVersion(crd *apiextensionsv1.CustomResourceDefinition, dyn
func isWatchCachePrimed(crd *apiextensionsv1.CustomResourceDefinition, dynamicClientSet dynamic.Interface) (bool, error) {
ns := ""
if crd.Spec.Scope != apiextensionsv1.ClusterScoped {
ns = "aval"
ns = "default"
}
versions := servedV1Versions(crd)

View File

@ -19,7 +19,6 @@ package integration
import (
"context"
"fmt"
"path"
"strings"
"testing"
"time"
@ -28,10 +27,8 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/util/json"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/apimachinery/pkg/util/yaml"
genericapirequest "k8s.io/apiserver/pkg/endpoints/request"
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
@ -701,7 +698,7 @@ func TestForbiddenFieldsInSchema(t *testing.T) {
}
func TestNonStructuralSchemaConditionUpdate(t *testing.T) {
tearDown, apiExtensionClient, _, etcdclient, etcdStoragePrefix, err := fixtures.StartDefaultServerWithClientsAndEtcd(t)
tearDown, apiExtensionClient, dynamicClient, etcdclient, etcdStoragePrefix, err := fixtures.StartDefaultServerWithClientsAndEtcd(t)
if err != nil {
t.Fatal(err)
}
@ -752,11 +749,8 @@ spec:
// create CRDs. We cannot create these in v1, but they can exist in upgraded clusters
t.Logf("Creating CRD %s", betaCRD.Name)
ctx := genericapirequest.WithNamespace(genericapirequest.NewContext(), metav1.NamespaceNone)
key := path.Join("/", etcdStoragePrefix, "apiextensions.k8s.io", "customresourcedefinitions/foos.tests.example.com")
val, _ := json.Marshal(betaCRD)
if _, err := etcdclient.Put(ctx, key, string(val)); err != nil {
t.Fatalf("unexpected error: %v", err)
if _, err := fixtures.CreateCRDUsingRemovedAPI(etcdclient, etcdStoragePrefix, betaCRD, apiExtensionClient, dynamicClient); err != nil {
t.Fatal(err)
}
// wait for condition with violations

View File

@ -62,7 +62,7 @@ func TestInternalVersionIsHandlerVersion(t *testing.T) {
t.Fatal(err)
}
// update validation via update because the cache priming in CreateNewCustomResourceDefinition will fail otherwise
// update validation via update because the cache priming in CreateCRDUsingRemovedAPI will fail otherwise
t.Logf("Updating CRD to validate apiVersion")
noxuDefinition, err = UpdateCustomResourceDefinitionWithRetry(apiExtensionClient, noxuDefinition.Name, func(crd *apiextensionsv1.CustomResourceDefinition) {
for i := range crd.Spec.Versions {

View File

@ -58,15 +58,15 @@ func TestApplyCRDNoSchema(t *testing.T) {
t.Fatal(err)
}
noxuDefinition := nearlyRemovedBetaMultipleVersionNoxuCRD(apiextensionsv1beta1.ClusterScoped)
noxuBetaDefinition := nearlyRemovedBetaMultipleVersionNoxuCRD(apiextensionsv1beta1.ClusterScoped)
noxuDefinition, err = fixtures.CreateNewCustomResourceDefinition(noxuDefinition, apiExtensionClient, dynamicClient)
noxuDefinition, err := fixtures.CreateCRDUsingRemovedAPI(server.EtcdClient, server.EtcdStoragePrefix, noxuBetaDefinition, apiExtensionClient, dynamicClient)
if err != nil {
t.Fatal(err)
}
kind := noxuDefinition.Spec.Names.Kind
apiVersion := noxuDefinition.Spec.Group + "/" + noxuDefinition.Spec.Version
apiVersion := noxuDefinition.Spec.Group + "/" + noxuDefinition.Spec.Versions[0].Name
name := "mytest"
rest := apiExtensionClient.Discovery().RESTClient()
@ -78,7 +78,7 @@ metadata:
spec:
replicas: 1`, apiVersion, kind, name))
result, err := rest.Patch(types.ApplyPatchType).
AbsPath("/apis", noxuDefinition.Spec.Group, noxuDefinition.Spec.Version, noxuDefinition.Spec.Names.Plural).
AbsPath("/apis", noxuDefinition.Spec.Group, noxuDefinition.Spec.Versions[0].Name, noxuDefinition.Spec.Names.Plural).
Name(name).
Param("fieldManager", "apply_test").
Body(yamlBody).
@ -90,7 +90,7 @@ spec:
// Patch object to change the number of replicas
result, err = rest.Patch(types.MergePatchType).
AbsPath("/apis", noxuDefinition.Spec.Group, noxuDefinition.Spec.Version, noxuDefinition.Spec.Names.Plural).
AbsPath("/apis", noxuDefinition.Spec.Group, noxuDefinition.Spec.Versions[0].Name, noxuDefinition.Spec.Names.Plural).
Name(name).
Body([]byte(`{"spec":{"replicas": 5}}`)).
DoRaw(context.TODO())
@ -101,7 +101,7 @@ spec:
// Re-apply, we should get conflicts now, since the number of replicas was changed.
result, err = rest.Patch(types.ApplyPatchType).
AbsPath("/apis", noxuDefinition.Spec.Group, noxuDefinition.Spec.Version, noxuDefinition.Spec.Names.Plural).
AbsPath("/apis", noxuDefinition.Spec.Group, noxuDefinition.Spec.Versions[0].Name, noxuDefinition.Spec.Names.Plural).
Name(name).
Param("fieldManager", "apply_test").
Body(yamlBody).
@ -119,7 +119,7 @@ spec:
// Re-apply with force, should work fine.
result, err = rest.Patch(types.ApplyPatchType).
AbsPath("/apis", noxuDefinition.Spec.Group, noxuDefinition.Spec.Version, noxuDefinition.Spec.Names.Plural).
AbsPath("/apis", noxuDefinition.Spec.Group, noxuDefinition.Spec.Versions[0].Name, noxuDefinition.Spec.Names.Plural).
Name(name).
Param("force", "true").
Param("fieldManager", "apply_test").
@ -157,7 +157,7 @@ spec:
}
}`)
result, err = rest.Patch(types.MergePatchType).
AbsPath("/apis", noxuDefinition.Spec.Group, noxuDefinition.Spec.Version, noxuDefinition.Spec.Names.Plural).
AbsPath("/apis", noxuDefinition.Spec.Group, noxuDefinition.Spec.Versions[0].Name, noxuDefinition.Spec.Names.Plural).
SubResource("status").
Name(name).
Param("fieldManager", "subresource_test").
@ -176,7 +176,7 @@ spec:
// However, it is possible to modify managed fields using the main resource
result, err = rest.Patch(types.MergePatchType).
AbsPath("/apis", noxuDefinition.Spec.Group, noxuDefinition.Spec.Version, noxuDefinition.Spec.Names.Plural).
AbsPath("/apis", noxuDefinition.Spec.Group, noxuDefinition.Spec.Versions[0].Name, noxuDefinition.Spec.Names.Plural).
Name(name).
Param("fieldManager", "subresource_test").
Body([]byte(`{"metadata":{"managedFields":[{}]}}`)).

View File

@ -22,9 +22,12 @@ import (
"testing"
"time"
"k8s.io/apiextensions-apiserver/test/integration/fixtures"
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
v1 "k8s.io/api/core/v1"
"k8s.io/api/policy/v1beta1"
apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
@ -105,7 +108,7 @@ func TestPDBWithScaleSubresource(t *testing.T) {
crdDefinition := newCustomResourceDefinition()
etcd.CreateTestCRDs(t, apiExtensionClient, true, crdDefinition)
gvr := schema.GroupVersionResource{Group: crdDefinition.Spec.Group, Version: crdDefinition.Spec.Version, Resource: crdDefinition.Spec.Names.Plural}
gvr := schema.GroupVersionResource{Group: crdDefinition.Spec.Group, Version: crdDefinition.Spec.Versions[0].Name, Resource: crdDefinition.Spec.Names.Plural}
resourceClient := dynamicClient.Resource(gvr).Namespace(nsName)
replicas := 4
@ -115,7 +118,7 @@ func TestPDBWithScaleSubresource(t *testing.T) {
resource := &unstructured.Unstructured{
Object: map[string]interface{}{
"kind": crdDefinition.Spec.Names.Kind,
"apiVersion": crdDefinition.Spec.Group + "/" + crdDefinition.Spec.Version,
"apiVersion": crdDefinition.Spec.Group + "/" + crdDefinition.Spec.Versions[0].Name,
"metadata": map[string]interface{}{
"name": "resource",
"namespace": nsName,
@ -134,7 +137,7 @@ func TestPDBWithScaleSubresource(t *testing.T) {
ownerRef := metav1.OwnerReference{
Name: resource.GetName(),
Kind: crdDefinition.Spec.Names.Kind,
APIVersion: crdDefinition.Spec.Group + "/" + crdDefinition.Spec.Version,
APIVersion: crdDefinition.Spec.Group + "/" + crdDefinition.Spec.Versions[0].Name,
UID: createdResource.GetUID(),
Controller: &trueValue,
}
@ -232,23 +235,30 @@ func addPodConditionReady(pod *v1.Pod) {
}
}
func newCustomResourceDefinition() *apiextensionsv1beta1.CustomResourceDefinition {
return &apiextensionsv1beta1.CustomResourceDefinition{
func newCustomResourceDefinition() *apiextensionsv1.CustomResourceDefinition {
return &apiextensionsv1.CustomResourceDefinition{
ObjectMeta: metav1.ObjectMeta{Name: "crds.mygroup.example.com"},
Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{
Group: "mygroup.example.com",
Version: "v1beta1",
Names: apiextensionsv1beta1.CustomResourceDefinitionNames{
Spec: apiextensionsv1.CustomResourceDefinitionSpec{
Group: "mygroup.example.com",
Names: apiextensionsv1.CustomResourceDefinitionNames{
Plural: "crds",
Singular: "crd",
Kind: "Crd",
ListKind: "CrdList",
},
Scope: apiextensionsv1beta1.NamespaceScoped,
Subresources: &apiextensionsv1beta1.CustomResourceSubresources{
Scale: &apiextensionsv1beta1.CustomResourceSubresourceScale{
SpecReplicasPath: ".spec.replicas",
StatusReplicasPath: ".status.replicas",
Scope: apiextensionsv1.NamespaceScoped,
Versions: []apiextensionsv1.CustomResourceDefinitionVersion{
{
Name: "v1beta1",
Served: true,
Storage: true,
Schema: fixtures.AllowAllSchema(),
Subresources: &apiextensionsv1.CustomResourceSubresources{
Scale: &apiextensionsv1.CustomResourceSubresourceScale{
SpecReplicasPath: ".spec.replicas",
StatusReplicasPath: ".status.replicas",
},
},
},
},
},

View File

@ -17,11 +17,11 @@ limitations under the License.
package etcd
import (
apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
"k8s.io/apiextensions-apiserver/test/integration/fixtures"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/kubernetes/test/utils/image"
"k8s.io/utils/pointer"
)
// GetEtcdStorageData returns etcd data for all persisted objects.
@ -574,36 +574,61 @@ type Prerequisite struct {
// GetCustomResourceDefinitionData returns the resource definitions that back the custom resources
// included in GetEtcdStorageData. They should be created using CreateTestCRDs before running any tests.
func GetCustomResourceDefinitionData() []*apiextensionsv1beta1.CustomResourceDefinition {
return []*apiextensionsv1beta1.CustomResourceDefinition{
// namespaced with legacy version field
// We can switch this to v1 CRDs based on transitive call site analysis.
// Call sites:
// 1. TestDedupOwnerReferences - beta doesn't matter
// 2. TestWebhookAdmissionWithWatchCache/TestWebhookAdmissionWithoutWatchCache - beta doesn't matter
// 3. TestApplyStatus - the version fields don't matter. Pruning isn't checked, just ownership.
// 4. TestDryRun - versions and pruning don't matter
// 5. TestStorageVersionBootstrap - versions and pruning don't matter.
// 6. TestEtcdStoragePath - beta doesn't matter
// 7. TestCrossGroupStorage - beta doesn't matter
// 8. TestOverlappingCustomResourceCustomResourceDefinition - beta doesn't matter
// 9. TestOverlappingCustomResourceAPIService - beta doesn't matter
func GetCustomResourceDefinitionData() []*apiextensionsv1.CustomResourceDefinition {
return []*apiextensionsv1.CustomResourceDefinition{
// namespaced
{
ObjectMeta: metav1.ObjectMeta{
Name: "foos.cr.bar.com",
},
Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{
Group: "cr.bar.com",
Version: "v1",
Scope: apiextensionsv1beta1.NamespaceScoped,
Names: apiextensionsv1beta1.CustomResourceDefinitionNames{
Spec: apiextensionsv1.CustomResourceDefinitionSpec{
Group: "cr.bar.com",
Scope: apiextensionsv1.NamespaceScoped,
Names: apiextensionsv1.CustomResourceDefinitionNames{
Plural: "foos",
Kind: "Foo",
},
Versions: []apiextensionsv1.CustomResourceDefinitionVersion{
{
Name: "v1",
Served: true,
Storage: true,
Schema: fixtures.AllowAllSchema(),
},
},
},
},
// cluster scoped with legacy version field
// cluster scoped
{
ObjectMeta: metav1.ObjectMeta{
Name: "pants.custom.fancy.com",
},
Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{
Group: "custom.fancy.com",
Version: "v2",
Scope: apiextensionsv1beta1.ClusterScoped,
Names: apiextensionsv1beta1.CustomResourceDefinitionNames{
Spec: apiextensionsv1.CustomResourceDefinitionSpec{
Group: "custom.fancy.com",
Scope: apiextensionsv1.ClusterScoped,
Names: apiextensionsv1.CustomResourceDefinitionNames{
Plural: "pants",
Kind: "Pant",
},
Versions: []apiextensionsv1.CustomResourceDefinitionVersion{
{
Name: "v2",
Served: true,
Storage: true,
Schema: fixtures.AllowAllSchema(),
},
},
},
},
// cluster scoped with legacy version field and pruning.
@ -611,25 +636,29 @@ func GetCustomResourceDefinitionData() []*apiextensionsv1beta1.CustomResourceDef
ObjectMeta: metav1.ObjectMeta{
Name: "integers.random.numbers.com",
},
Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{
Group: "random.numbers.com",
Version: "v1",
Scope: apiextensionsv1beta1.ClusterScoped,
Names: apiextensionsv1beta1.CustomResourceDefinitionNames{
Spec: apiextensionsv1.CustomResourceDefinitionSpec{
Group: "random.numbers.com",
Scope: apiextensionsv1.ClusterScoped,
Names: apiextensionsv1.CustomResourceDefinitionNames{
Plural: "integers",
Kind: "Integer",
},
Validation: &apiextensionsv1beta1.CustomResourceValidation{
OpenAPIV3Schema: &apiextensionsv1beta1.JSONSchemaProps{
Type: "object",
Properties: map[string]apiextensionsv1beta1.JSONSchemaProps{
"value": {
Type: "number",
},
},
Versions: []apiextensionsv1.CustomResourceDefinitionVersion{
{
Name: "v1",
Served: true,
Storage: true,
Schema: &apiextensionsv1.CustomResourceValidation{
OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{
Type: "object",
Properties: map[string]apiextensionsv1.JSONSchemaProps{
"value": {
Type: "number",
},
},
}},
},
},
PreserveUnknownFields: pointer.BoolPtr(false),
},
},
// cluster scoped with versions field
@ -637,38 +666,57 @@ func GetCustomResourceDefinitionData() []*apiextensionsv1beta1.CustomResourceDef
ObjectMeta: metav1.ObjectMeta{
Name: "pandas.awesome.bears.com",
},
Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{
Spec: apiextensionsv1.CustomResourceDefinitionSpec{
Group: "awesome.bears.com",
Versions: []apiextensionsv1beta1.CustomResourceDefinitionVersion{
Versions: []apiextensionsv1.CustomResourceDefinitionVersion{
{
Name: "v1",
Served: true,
Storage: true,
Schema: fixtures.AllowAllSchema(),
Subresources: &apiextensionsv1.CustomResourceSubresources{
Status: &apiextensionsv1.CustomResourceSubresourceStatus{},
Scale: &apiextensionsv1.CustomResourceSubresourceScale{
SpecReplicasPath: ".spec.replicas",
StatusReplicasPath: ".status.replicas",
LabelSelectorPath: func() *string { path := ".status.selector"; return &path }(),
},
},
},
{
Name: "v2",
Served: false,
Storage: false,
Schema: fixtures.AllowAllSchema(),
Subresources: &apiextensionsv1.CustomResourceSubresources{
Status: &apiextensionsv1.CustomResourceSubresourceStatus{},
Scale: &apiextensionsv1.CustomResourceSubresourceScale{
SpecReplicasPath: ".spec.replicas",
StatusReplicasPath: ".status.replicas",
LabelSelectorPath: func() *string { path := ".status.selector"; return &path }(),
},
},
},
{
Name: "v3",
Served: true,
Storage: false,
Schema: fixtures.AllowAllSchema(),
Subresources: &apiextensionsv1.CustomResourceSubresources{
Status: &apiextensionsv1.CustomResourceSubresourceStatus{},
Scale: &apiextensionsv1.CustomResourceSubresourceScale{
SpecReplicasPath: ".spec.replicas",
StatusReplicasPath: ".status.replicas",
LabelSelectorPath: func() *string { path := ".status.selector"; return &path }(),
},
},
},
},
Scope: apiextensionsv1beta1.ClusterScoped,
Names: apiextensionsv1beta1.CustomResourceDefinitionNames{
Scope: apiextensionsv1.ClusterScoped,
Names: apiextensionsv1.CustomResourceDefinitionNames{
Plural: "pandas",
Kind: "Panda",
},
Subresources: &apiextensionsv1beta1.CustomResourceSubresources{
Status: &apiextensionsv1beta1.CustomResourceSubresourceStatus{},
Scale: &apiextensionsv1beta1.CustomResourceSubresourceScale{
SpecReplicasPath: ".spec.replicas",
StatusReplicasPath: ".status.replicas",
LabelSelectorPath: func() *string { path := ".status.selector"; return &path }(),
},
},
},
},
}

View File

@ -30,7 +30,7 @@ import (
"go.etcd.io/etcd/clientv3"
"go.etcd.io/etcd/clientv3/concurrency"
apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@ -303,14 +303,14 @@ func JSONToUnstructured(stub, namespace string, mapping *meta.RESTMapping, dynam
// CreateTestCRDs creates the given CRDs, any failure causes the test to Fatal.
// If skipCrdExistsInDiscovery is true, the CRDs are only checked for the Established condition via their Status.
// If skipCrdExistsInDiscovery is false, the CRDs are checked via discovery, see CrdExistsInDiscovery.
func CreateTestCRDs(t *testing.T, client apiextensionsclientset.Interface, skipCrdExistsInDiscovery bool, crds ...*apiextensionsv1beta1.CustomResourceDefinition) {
func CreateTestCRDs(t *testing.T, client apiextensionsclientset.Interface, skipCrdExistsInDiscovery bool, crds ...*apiextensionsv1.CustomResourceDefinition) {
for _, crd := range crds {
createTestCRD(t, client, skipCrdExistsInDiscovery, crd)
}
}
func createTestCRD(t *testing.T, client apiextensionsclientset.Interface, skipCrdExistsInDiscovery bool, crd *apiextensionsv1beta1.CustomResourceDefinition) {
if _, err := client.ApiextensionsV1beta1().CustomResourceDefinitions().Create(context.TODO(), crd, metav1.CreateOptions{}); err != nil {
func createTestCRD(t *testing.T, client apiextensionsclientset.Interface, skipCrdExistsInDiscovery bool, crd *apiextensionsv1.CustomResourceDefinition) {
if _, err := client.ApiextensionsV1().CustomResourceDefinitions().Create(context.TODO(), crd, metav1.CreateOptions{}); err != nil {
t.Fatalf("Failed to create %s CRD; %v", crd.Name, err)
}
if skipCrdExistsInDiscovery {
@ -328,14 +328,14 @@ func createTestCRD(t *testing.T, client apiextensionsclientset.Interface, skipCr
func waitForEstablishedCRD(client apiextensionsclientset.Interface, name string) error {
return wait.PollImmediate(500*time.Millisecond, wait.ForeverTestTimeout, func() (bool, error) {
crd, err := client.ApiextensionsV1beta1().CustomResourceDefinitions().Get(context.TODO(), name, metav1.GetOptions{})
crd, err := client.ApiextensionsV1().CustomResourceDefinitions().Get(context.TODO(), name, metav1.GetOptions{})
if err != nil {
return false, err
}
for _, cond := range crd.Status.Conditions {
switch cond.Type {
case apiextensionsv1beta1.Established:
if cond.Status == apiextensionsv1beta1.ConditionTrue {
case apiextensionsv1.Established:
if cond.Status == apiextensionsv1.ConditionTrue {
return true, nil
}
}
@ -345,11 +345,8 @@ func waitForEstablishedCRD(client apiextensionsclientset.Interface, name string)
}
// CrdExistsInDiscovery checks to see if the given CRD exists in discovery at all served versions.
func CrdExistsInDiscovery(client apiextensionsclientset.Interface, crd *apiextensionsv1beta1.CustomResourceDefinition) bool {
func CrdExistsInDiscovery(client apiextensionsclientset.Interface, crd *apiextensionsv1.CustomResourceDefinition) bool {
var versions []string
if len(crd.Spec.Version) != 0 {
versions = append(versions, crd.Spec.Version)
}
for _, v := range crd.Spec.Versions {
if v.Served {
versions = append(versions, v.Name)
@ -363,7 +360,7 @@ func CrdExistsInDiscovery(client apiextensionsclientset.Interface, crd *apiexten
return true
}
func crdVersionExistsInDiscovery(client apiextensionsclientset.Interface, crd *apiextensionsv1beta1.CustomResourceDefinition, version string) bool {
func crdVersionExistsInDiscovery(client apiextensionsclientset.Interface, crd *apiextensionsv1.CustomResourceDefinition, version string) bool {
resourceList, err := client.Discovery().ServerResourcesForGroupVersion(crd.Spec.Group + "/" + version)
if err != nil {
return false

View File

@ -226,7 +226,7 @@ func setupWithServer(t *testing.T, result *kubeapiservertesting.TestServer, work
if err != nil {
t.Fatalf("error creating extension clientset: %v", err)
}
// CreateNewCustomResourceDefinition wants to use this namespace for verifying
// CreateCRDUsingRemovedAPI wants to use this namespace for verifying
// namespace-scoped CRD creation.
createNamespaceOrDie("aval", clientSet, t)

View File

@ -24,6 +24,10 @@ import (
"testing"
"time"
"k8s.io/apiextensions-apiserver/test/integration/fixtures"
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
"github.com/go-openapi/spec"
v1 "k8s.io/api/core/v1"
@ -38,7 +42,6 @@ import (
kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing"
"k8s.io/kubernetes/test/integration/etcd"
"k8s.io/kubernetes/test/integration/framework"
utilpointer "k8s.io/utils/pointer"
)
func TestCRDShadowGroup(t *testing.T) {
@ -72,18 +75,26 @@ func TestCRDShadowGroup(t *testing.T) {
}
t.Logf("Trying to shadow networking group")
crd := &apiextensionsv1beta1.CustomResourceDefinition{
crd := &apiextensionsv1.CustomResourceDefinition{
ObjectMeta: metav1.ObjectMeta{
Name: "foos." + networkingv1.GroupName,
Name: "foos." + networkingv1.GroupName,
Annotations: map[string]string{"api-approved.kubernetes.io": "unapproved, test-only"},
},
Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{
Group: networkingv1.GroupName,
Version: networkingv1.SchemeGroupVersion.Version,
Scope: apiextensionsv1beta1.ClusterScoped,
Names: apiextensionsv1beta1.CustomResourceDefinitionNames{
Spec: apiextensionsv1.CustomResourceDefinitionSpec{
Group: networkingv1.GroupName,
Scope: apiextensionsv1.ClusterScoped,
Names: apiextensionsv1.CustomResourceDefinitionNames{
Plural: "foos",
Kind: "Foo",
},
Versions: []apiextensionsv1.CustomResourceDefinitionVersion{
{
Name: networkingv1.SchemeGroupVersion.Version,
Served: true,
Storage: true,
Schema: fixtures.AllowAllSchema(),
},
},
},
}
etcd.CreateTestCRDs(t, apiextensionsclient, true, crd)
@ -122,18 +133,25 @@ func TestCRD(t *testing.T) {
}
t.Logf("Trying to create a custom resource without conflict")
crd := &apiextensionsv1beta1.CustomResourceDefinition{
crd := &apiextensionsv1.CustomResourceDefinition{
ObjectMeta: metav1.ObjectMeta{
Name: "foos.cr.bar.com",
},
Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{
Group: "cr.bar.com",
Version: "v1",
Scope: apiextensionsv1beta1.NamespaceScoped,
Names: apiextensionsv1beta1.CustomResourceDefinitionNames{
Spec: apiextensionsv1.CustomResourceDefinitionSpec{
Group: "cr.bar.com",
Scope: apiextensionsv1.NamespaceScoped,
Names: apiextensionsv1.CustomResourceDefinitionNames{
Plural: "foos",
Kind: "Foo",
},
Versions: []apiextensionsv1.CustomResourceDefinitionVersion{
{
Name: networkingv1.SchemeGroupVersion.Version,
Served: true,
Storage: true,
Schema: fixtures.AllowAllSchema(),
},
},
},
}
etcd.CreateTestCRDs(t, apiextensionsclient, false, crd)
@ -161,9 +179,13 @@ func TestCRDOpenAPI(t *testing.T) {
if err != nil {
t.Fatalf("Unexpected error: %v", err)
}
dynamicClient, err := dynamic.NewForConfig(result.ClientConfig)
if err != nil {
t.Fatalf("Unexpected error: %v", err)
}
t.Logf("Trying to create a CustomResourceDefinitions")
nonStructuralCRD := &apiextensionsv1beta1.CustomResourceDefinition{
nonStructuralBetaCRD := &apiextensionsv1beta1.CustomResourceDefinition{
ObjectMeta: metav1.ObjectMeta{
Name: "foos.nonstructural.cr.bar.com",
},
@ -185,30 +207,38 @@ func TestCRDOpenAPI(t *testing.T) {
},
},
}
structuralCRD := &apiextensionsv1beta1.CustomResourceDefinition{
structuralCRD := &apiextensionsv1.CustomResourceDefinition{
ObjectMeta: metav1.ObjectMeta{
Name: "foos.structural.cr.bar.com",
},
Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{
Group: "structural.cr.bar.com",
Version: "v1",
Scope: apiextensionsv1beta1.NamespaceScoped,
Names: apiextensionsv1beta1.CustomResourceDefinitionNames{
Spec: apiextensionsv1.CustomResourceDefinitionSpec{
Group: "structural.cr.bar.com",
Scope: apiextensionsv1.NamespaceScoped,
Names: apiextensionsv1.CustomResourceDefinitionNames{
Plural: "foos",
Kind: "Foo",
},
PreserveUnknownFields: utilpointer.BoolPtr(false),
Validation: &apiextensionsv1beta1.CustomResourceValidation{
OpenAPIV3Schema: &apiextensionsv1beta1.JSONSchemaProps{
Type: "object",
Properties: map[string]apiextensionsv1beta1.JSONSchemaProps{
"foo": {Type: "string"},
Versions: []apiextensionsv1.CustomResourceDefinitionVersion{
{
Name: "v1",
Served: true,
Storage: true,
Schema: &apiextensionsv1.CustomResourceValidation{
OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{
Type: "object",
Properties: map[string]apiextensionsv1.JSONSchemaProps{
"foo": {Type: "string"},
},
},
},
},
},
},
}
etcd.CreateTestCRDs(t, apiextensionsclient, false, nonStructuralCRD)
nonStructuralCRD, err := fixtures.CreateCRDUsingRemovedAPI(result.EtcdClient, result.EtcdStoragePrefix, nonStructuralBetaCRD, apiextensionsclient, dynamicClient)
if err != nil {
t.Fatal(err)
}
etcd.CreateTestCRDs(t, apiextensionsclient, false, structuralCRD)
getPublishedSchema := func(defName string) (*spec.Schema, error) {
@ -230,7 +260,7 @@ func TestCRDOpenAPI(t *testing.T) {
return &d, nil
}
waitForSpec := func(crd *apiextensionsv1beta1.CustomResourceDefinition, expectedType string) {
waitForSpec := func(crd *apiextensionsv1.CustomResourceDefinition, expectedType string) {
t.Logf(`Waiting for {properties: {"foo": {"type":"%s"}}} to show up in schema`, expectedType)
lastMsg := ""
if err := wait.PollImmediate(500*time.Millisecond, 10*time.Second, func() (bool, error) {
@ -262,14 +292,14 @@ func TestCRDOpenAPI(t *testing.T) {
t.Logf("Check that structural schema is published")
waitForSpec(structuralCRD, "string")
structuralCRD, err = apiextensionsclient.ApiextensionsV1beta1().CustomResourceDefinitions().Get(context.TODO(), structuralCRD.Name, metav1.GetOptions{})
structuralCRD, err = apiextensionsclient.ApiextensionsV1().CustomResourceDefinitions().Get(context.TODO(), structuralCRD.Name, metav1.GetOptions{})
if err != nil {
t.Fatal(err)
}
prop := structuralCRD.Spec.Validation.OpenAPIV3Schema.Properties["foo"]
prop := structuralCRD.Spec.Versions[0].Schema.OpenAPIV3Schema.Properties["foo"]
prop.Type = "boolean"
structuralCRD.Spec.Validation.OpenAPIV3Schema.Properties["foo"] = prop
if _, err = apiextensionsclient.ApiextensionsV1beta1().CustomResourceDefinitions().Update(context.TODO(), structuralCRD, metav1.UpdateOptions{}); err != nil {
structuralCRD.Spec.Versions[0].Schema.OpenAPIV3Schema.Properties["foo"] = prop
if _, err = apiextensionsclient.ApiextensionsV1().CustomResourceDefinitions().Update(context.TODO(), structuralCRD, metav1.UpdateOptions{}); err != nil {
t.Fatal(err)
}
waitForSpec(structuralCRD, "boolean")
@ -287,10 +317,10 @@ func TestCRDOpenAPI(t *testing.T) {
}
}
func crdDefinitionName(crd *apiextensionsv1beta1.CustomResourceDefinition) string {
func crdDefinitionName(crd *apiextensionsv1.CustomResourceDefinition) string {
sgmts := strings.Split(crd.Spec.Group, ".")
reverse(sgmts)
return strings.Join(append(sgmts, crd.Spec.Version, crd.Spec.Names.Kind), ".")
return strings.Join(append(sgmts, crd.Spec.Versions[0].Name, crd.Spec.Names.Kind), ".")
}
func reverse(s []string) {

View File

@ -28,6 +28,10 @@ import (
"testing"
"time"
"k8s.io/apiextensions-apiserver/test/integration/fixtures"
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
"github.com/go-openapi/spec"
appsv1 "k8s.io/api/apps/v1"
@ -204,26 +208,32 @@ func TestOpenAPIApiextensionsOverlapProtection(t *testing.T) {
}
// Create a CRD that overlaps OpenAPI path with the CRD API
crd := &apiextensionsv1beta1.CustomResourceDefinition{
crd := &apiextensionsv1.CustomResourceDefinition{
ObjectMeta: metav1.ObjectMeta{
Name: "customresourcedefinitions.apiextensions.k8s.io",
Annotations: map[string]string{"api-approved.kubernetes.io": "unapproved, test-only"},
},
Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{
Group: "apiextensions.k8s.io",
Version: "v1beta1",
Scope: apiextensionsv1beta1.ClusterScoped,
Names: apiextensionsv1beta1.CustomResourceDefinitionNames{
Spec: apiextensionsv1.CustomResourceDefinitionSpec{
Group: "apiextensions.k8s.io",
Scope: apiextensionsv1.ClusterScoped,
Names: apiextensionsv1.CustomResourceDefinitionNames{
Plural: "customresourcedefinitions",
Singular: "customresourcedefinition",
Kind: "CustomResourceDefinition",
ListKind: "CustomResourceDefinitionList",
},
Validation: &apiextensionsv1beta1.CustomResourceValidation{
OpenAPIV3Schema: &apiextensionsv1beta1.JSONSchemaProps{
Type: "object",
Properties: map[string]apiextensionsv1beta1.JSONSchemaProps{
testApiextensionsOverlapProbeString: {Type: "boolean"},
Versions: []apiextensionsv1.CustomResourceDefinitionVersion{
{
Name: "v1beta1",
Served: true,
Storage: true,
Schema: &apiextensionsv1.CustomResourceValidation{
OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{
Type: "object",
Properties: map[string]apiextensionsv1.JSONSchemaProps{
testApiextensionsOverlapProbeString: {Type: "boolean"},
},
},
},
},
},
@ -263,26 +273,32 @@ func TestOpenAPIApiextensionsOverlapProtection(t *testing.T) {
}
// Create a CRD that overlaps OpenAPI definition with the CRD API
crd = &apiextensionsv1beta1.CustomResourceDefinition{
crd = &apiextensionsv1.CustomResourceDefinition{
ObjectMeta: metav1.ObjectMeta{
Name: "customresourcedefinitions.apiextensions.apis.pkg.apiextensions-apiserver.k8s.io",
Annotations: map[string]string{"api-approved.kubernetes.io": "unapproved, test-only"},
},
Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{
Group: "apiextensions.apis.pkg.apiextensions-apiserver.k8s.io",
Version: "v1beta1",
Scope: apiextensionsv1beta1.ClusterScoped,
Names: apiextensionsv1beta1.CustomResourceDefinitionNames{
Spec: apiextensionsv1.CustomResourceDefinitionSpec{
Group: "apiextensions.apis.pkg.apiextensions-apiserver.k8s.io",
Scope: apiextensionsv1.ClusterScoped,
Names: apiextensionsv1.CustomResourceDefinitionNames{
Plural: "customresourcedefinitions",
Singular: "customresourcedefinition",
Kind: "CustomResourceDefinition",
ListKind: "CustomResourceDefinitionList",
},
Validation: &apiextensionsv1beta1.CustomResourceValidation{
OpenAPIV3Schema: &apiextensionsv1beta1.JSONSchemaProps{
Type: "object",
Properties: map[string]apiextensionsv1beta1.JSONSchemaProps{
testApiextensionsOverlapProbeString: {Type: "boolean"},
Versions: []apiextensionsv1.CustomResourceDefinitionVersion{
{
Name: "v1beta1",
Served: true,
Storage: true,
Schema: &apiextensionsv1.CustomResourceValidation{
OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{
Type: "object",
Properties: map[string]apiextensionsv1.JSONSchemaProps{
testApiextensionsOverlapProbeString: {Type: "boolean"},
},
},
},
},
},
@ -319,18 +335,25 @@ func triggerSpecUpdateWithProbeCRD(t *testing.T, apiextensionsclient *apiextensi
name := fmt.Sprintf("integration-test-%s-crd", suffix)
kind := fmt.Sprintf("Integration-test-%s-crd", suffix)
group := "probe.test.com"
crd := &apiextensionsv1beta1.CustomResourceDefinition{
crd := &apiextensionsv1.CustomResourceDefinition{
ObjectMeta: metav1.ObjectMeta{Name: name + "s." + group},
Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{
Group: group,
Version: "v1",
Scope: apiextensionsv1beta1.ClusterScoped,
Names: apiextensionsv1beta1.CustomResourceDefinitionNames{
Spec: apiextensionsv1.CustomResourceDefinitionSpec{
Group: group,
Scope: apiextensionsv1.ClusterScoped,
Names: apiextensionsv1.CustomResourceDefinitionNames{
Plural: name + "s",
Singular: name,
Kind: kind,
ListKind: kind + "List",
},
Versions: []apiextensionsv1.CustomResourceDefinitionVersion{
{
Name: "v1",
Served: true,
Storage: true,
Schema: fixtures.AllowAllSchema(),
},
},
},
}
etcd.CreateTestCRDs(t, apiextensionsclient, false, crd)

View File

@ -88,7 +88,7 @@ func testBuiltinResourceWrite(t *testing.T, cfg *rest.Config, shouldBlock bool)
func testCRDWrite(t *testing.T, cfg *rest.Config, shouldBlock bool) {
crdClient := apiextensionsclientset.NewForConfigOrDie(cfg)
_, err := crdClient.ApiextensionsV1beta1().CustomResourceDefinitions().Create(context.TODO(), etcd.GetCustomResourceDefinitionData()[1], metav1.CreateOptions{})
_, err := crdClient.ApiextensionsV1().CustomResourceDefinitions().Create(context.TODO(), etcd.GetCustomResourceDefinitionData()[1], metav1.CreateOptions{})
assertBlocking("writes to CRD", t, err, shouldBlock)
}