Merge pull request #130276 from stlaz/svm_flakes

Fix SVM test flaking because of occasional slow resource storage update
This commit is contained in:
Kubernetes Prow Robot 2025-03-20 03:52:31 -07:00 committed by GitHub
commit 08570c779b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 53 additions and 42 deletions

View File

@ -28,15 +28,17 @@ import (
svmv1alpha1 "k8s.io/api/storagemigration/v1alpha1" svmv1alpha1 "k8s.io/api/storagemigration/v1alpha1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/util/wait"
encryptionconfigcontroller "k8s.io/apiserver/pkg/server/options/encryptionconfig/controller" encryptionconfigcontroller "k8s.io/apiserver/pkg/server/options/encryptionconfig/controller"
etcd3watcher "k8s.io/apiserver/pkg/storage/etcd3" etcd3watcher "k8s.io/apiserver/pkg/storage/etcd3"
utilfeature "k8s.io/apiserver/pkg/util/feature" utilfeature "k8s.io/apiserver/pkg/util/feature"
clientgofeaturegate "k8s.io/client-go/features" clientgofeaturegate "k8s.io/client-go/features"
"k8s.io/component-base/featuregate" "k8s.io/component-base/featuregate"
featuregatetesting "k8s.io/component-base/featuregate/testing" featuregatetesting "k8s.io/component-base/featuregate/testing"
"k8s.io/klog/v2/ktesting"
"k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/features"
"k8s.io/kubernetes/test/integration/framework" "k8s.io/kubernetes/test/integration/framework"
"k8s.io/kubernetes/test/utils/ktesting"
) )
// TestStorageVersionMigration is an integration test that verifies storage version migration works. // TestStorageVersionMigration is an integration test that verifies storage version migration works.
@ -56,9 +58,7 @@ func TestStorageVersionMigration(t *testing.T) {
// this makes the test super responsive. It's set to a default of 1 minute. // this makes the test super responsive. It's set to a default of 1 minute.
encryptionconfigcontroller.EncryptionConfigFileChangePollDuration = time.Second encryptionconfigcontroller.EncryptionConfigFileChangePollDuration = time.Second
_, ctx := ktesting.NewTestContext(t) ctx := ktesting.Init(t)
ctx, cancel := context.WithCancel(ctx)
defer cancel()
svmTest := svmSetup(ctx, t) svmTest := svmSetup(ctx, t)
@ -92,7 +92,7 @@ func TestStorageVersionMigration(t *testing.T) {
} }
wantPrefix := "k8s:enc:aescbc:v1:key2" wantPrefix := "k8s:enc:aescbc:v1:key2"
etcdSecret, err := svmTest.getRawSecretFromETCD(t, secret.Name, secret.Namespace) etcdSecret, err := svmTest.getRawSecretFromETCD(t, secret.Namespace, secret.Name)
if err != nil { if err != nil {
t.Fatalf("Failed to get secret from etcd: %v", err) t.Fatalf("Failed to get secret from etcd: %v", err)
} }
@ -163,9 +163,7 @@ func TestStorageVersionMigrationWithCRD(t *testing.T) {
goleak.IgnoreTopFunction("github.com/moby/spdystream.(*Connection).shutdown"), goleak.IgnoreTopFunction("github.com/moby/spdystream.(*Connection).shutdown"),
) )
_, ctx := ktesting.NewTestContext(t) ctx := ktesting.Init(t)
ctx, cancel := context.WithCancel(ctx)
defer cancel()
crVersions := make(map[string]versions) crVersions := make(map[string]versions)
@ -210,10 +208,24 @@ func TestStorageVersionMigrationWithCRD(t *testing.T) {
svmTest.updateCRD(ctx, t, crd.Name, v2StorageCRDVersion, []string{"v1", "v2"}, "v2") svmTest.updateCRD(ctx, t, crd.Name, v2StorageCRDVersion, []string{"v1", "v2"}, "v2")
// create CR with v1 // create CR with v1
cr3 := svmTest.createCR(ctx, t, "cr3", "v1") var cr3 *unstructured.Unstructured
if ok := svmTest.isCRStoredAtVersion(t, "v2", cr3.GetName()); !ok { // updateCRD checks discovery returns storageVersionHash matching storage version v2
t.Fatalf("CR not stored at version v2") // to make sure the API server uses v2 but CRD controllers may race and the resource
// might still get stored in v1.
// Attempt to recreate the CR until it gets stored as v2.
// https://github.com/kubernetes/kubernetes/issues/130235
err := wait.PollUntilContextTimeout(ctx, 1*time.Second, 10*time.Second, true, func(waitCtx context.Context) (done bool, err error) {
cr3 = svmTest.createCR(waitCtx, t, "cr3", "v1")
if ok := svmTest.isCRStoredAtVersion(t, "v2", cr3.GetName()); !ok {
svmTest.deleteCR(waitCtx, t, cr3.GetName(), "v1")
return false, nil
}
return true, nil
})
if err != nil {
t.Fatalf("timed out waiting for CR to be stored as v2: %v", err)
} }
crVersions[cr3.GetName()] = versions{ crVersions[cr3.GetName()] = versions{
generation: cr3.GetGeneration(), generation: cr3.GetGeneration(),
rv: cr3.GetResourceVersion(), rv: cr3.GetResourceVersion(),
@ -291,9 +303,7 @@ func TestStorageVersionMigrationDuringChaos(t *testing.T) {
featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StorageVersionMigrator, true) featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StorageVersionMigrator, true)
featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, featuregate.Feature(clientgofeaturegate.InformerResourceVersion), true) featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, featuregate.Feature(clientgofeaturegate.InformerResourceVersion), true)
_, ctx := ktesting.NewTestContext(t) ctx := ktesting.Init(t)
ctx, cancel := context.WithCancel(ctx)
t.Cleanup(cancel)
svmTest := svmSetup(ctx, t) svmTest := svmSetup(ctx, t)

View File

@ -388,9 +388,9 @@ func (svm *svmTest) createSecret(ctx context.Context, t *testing.T, name, namesp
return svm.client.CoreV1().Secrets(secret.Namespace).Create(ctx, secret, metav1.CreateOptions{}) return svm.client.CoreV1().Secrets(secret.Namespace).Create(ctx, secret, metav1.CreateOptions{})
} }
func (svm *svmTest) getRawSecretFromETCD(t *testing.T, name, namespace string) ([]byte, error) { func (svm *svmTest) getRawSecretFromETCD(t *testing.T, namespace, name string) ([]byte, error) {
t.Helper() t.Helper()
secretETCDPath := svm.getETCDPathForResource(t, svm.storageConfig.Prefix, "", "secrets", name, namespace) secretETCDPath := getETCDPathForResource(t, svm.storageConfig.Prefix, "", "secrets", namespace, name)
etcdResponse, err := svm.readRawRecordFromETCD(t, secretETCDPath) etcdResponse, err := svm.readRawRecordFromETCD(t, secretETCDPath)
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to read %s from etcd: %w", secretETCDPath, err) return nil, fmt.Errorf("failed to read %s from etcd: %w", secretETCDPath, err)
@ -398,7 +398,7 @@ func (svm *svmTest) getRawSecretFromETCD(t *testing.T, name, namespace string) (
return etcdResponse.Kvs[0].Value, nil return etcdResponse.Kvs[0].Value, nil
} }
func (svm *svmTest) getETCDPathForResource(t *testing.T, storagePrefix, group, resource, name, namespaceName string) string { func getETCDPathForResource(t *testing.T, storagePrefix, group, resource, namespaceName, name string) string {
t.Helper() t.Helper()
groupResource := resource groupResource := resource
if group != "" { if group != "" {
@ -432,9 +432,9 @@ func (svm *svmTest) readRawRecordFromETCD(t *testing.T, path string) (*clientv3.
return response, nil return response, nil
} }
func (svm *svmTest) getRawCRFromETCD(t *testing.T, name, namespace, crdGroup, crdName string) ([]byte, error) { func (svm *svmTest) getRawCRFromETCD(t *testing.T, crdGroup, crdName, namespace, name string) ([]byte, error) {
t.Helper() t.Helper()
crdETCDPath := svm.getETCDPathForResource(t, svm.storageConfig.Prefix, crdGroup, crdName, name, namespace) crdETCDPath := getETCDPathForResource(t, svm.storageConfig.Prefix, crdGroup, crdName, namespace, name)
etcdResponse, err := svm.readRawRecordFromETCD(t, crdETCDPath) etcdResponse, err := svm.readRawRecordFromETCD(t, crdETCDPath)
if err != nil { if err != nil {
t.Fatalf("failed to read %s from etcd: %v", crdETCDPath, err) t.Fatalf("failed to read %s from etcd: %v", crdETCDPath, err)
@ -808,29 +808,30 @@ func (svm *svmTest) waitForCRDUpdate(
return false, fmt.Errorf("failed to get server groups and resources: %w", err) return false, fmt.Errorf("failed to get server groups and resources: %w", err)
} }
for _, api := range apiGroups { for _, api := range apiGroups {
if api.Name == crdGroup { if api.Name != crdGroup {
var servingVersions []string continue
for _, apiVersion := range api.Versions { }
servingVersions = append(servingVersions, apiVersion.Version) var servingVersions []string
} for _, apiVersion := range api.Versions {
sort.Strings(servingVersions) servingVersions = append(servingVersions, apiVersion.Version)
}
sort.Strings(servingVersions)
// Check if the serving versions are as expected // Check if the serving versions are as expected
if reflect.DeepEqual(expectedServingVersions, servingVersions) { if !reflect.DeepEqual(expectedServingVersions, servingVersions) {
expectedHash := endpointsdiscovery.StorageVersionHash(crdGroup, expectedStorageVersion, crdKind) continue
resourceList, err := svm.discoveryClient.ServerResourcesForGroupVersion(crdGroup + "/" + api.PreferredVersion.Version) }
if err != nil {
return false, fmt.Errorf("failed to get server resources for group version: %w", err)
}
// Check if the storage version is as expected expectedHash := endpointsdiscovery.StorageVersionHash(crdGroup, expectedStorageVersion, crdKind)
for _, resource := range resourceList.APIResources { resourceList, err := svm.discoveryClient.ServerResourcesForGroupVersion(crdGroup + "/" + api.PreferredVersion.Version)
if resource.Kind == crdKind { if err != nil {
if resource.StorageVersionHash == expectedHash { return false, fmt.Errorf("failed to get server resources for group version: %w", err)
return true, nil }
}
} // Check if the storage version is as expected
} for _, resource := range resourceList.APIResources {
if resource.Kind == crdKind && resource.StorageVersionHash == expectedHash {
return true, nil
} }
} }
} }
@ -1056,7 +1057,7 @@ func (svm *svmTest) setupServerCert(t *testing.T) *certContext {
func (svm *svmTest) isCRStoredAtVersion(t *testing.T, version, crName string) bool { func (svm *svmTest) isCRStoredAtVersion(t *testing.T, version, crName string) bool {
t.Helper() t.Helper()
data, err := svm.getRawCRFromETCD(t, crName, defaultNamespace, crdGroup, crdName+"s") data, err := svm.getRawCRFromETCD(t, crdGroup, crdName+"s", defaultNamespace, crName)
if err != nil { if err != nil {
t.Fatalf("Failed to get CR from etcd: %v", err) t.Fatalf("Failed to get CR from etcd: %v", err)
} }
@ -1135,7 +1136,7 @@ func (svm *svmTest) validateRVAndGeneration(ctx context.Context, t *testing.T, c
for crName, version := range crVersions { for crName, version := range crVersions {
// get CR from etcd // get CR from etcd
data, err := svm.getRawCRFromETCD(t, crName, defaultNamespace, crdGroup, crdName+"s") data, err := svm.getRawCRFromETCD(t, crdGroup, crdName+"s", defaultNamespace, crName)
if err != nil { if err != nil {
t.Fatalf("Failed to get CR from etcd: %v", err) t.Fatalf("Failed to get CR from etcd: %v", err)
} }