github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-08-10 04:27:54 +00:00
Code Issues Projects Releases Wiki Activity

expand etcd storage tests to test for 1.31..1.33

Browse Source
This commit is contained in:
Jefftree 2024-12-10 16:21:52 +00:00
parent dc476e968d
commit 6c94adcc63
4 changed files with 119 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
test/integration/controlplane/transformation/kms_transformation_test.go
View File

@ -618,9 +618,6 @@ resources:
endpoint: unix:///@encrypt-all-kms-provider.sock
`
// KUBE_APISERVER_SERVE_REMOVED_APIS_FOR_ONE_RELEASE allows for APIs pending removal to not block tests
t.Setenv("KUBE_APISERVER_SERVE_REMOVED_APIS_FOR_ONE_RELEASE", "true")
t.Run("encrypt all resources", func(t *testing.T) {
_ = mock.NewBase64Plugin(t, "@encrypt-all-kms-provider.sock")
// To ensure we are checking all REST resources

80
test/integration/etcd/data.go
View File

@ -17,26 +17,54 @@ limitations under the License.
package etcd
import (
"strings"
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
"k8s.io/apiextensions-apiserver/test/integration/fixtures"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime/schema"
utilversion "k8s.io/component-base/version"
"k8s.io/kubernetes/test/utils/image"
)
// GetEtcdStorageData returns etcd data for all persisted objects.
// GetSupportedEmulatedVersions provides the list of supported emulated versions in the etcd data.
// Tests aiming for full coverage of versions should test fixtures of all supported versions.
func GetSupportedEmulatedVersions() []string {
return []string{
utilversion.DefaultKubeEffectiveVersion().BinaryVersion().SubtractMinor(2).String(),
utilversion.DefaultKubeEffectiveVersion().BinaryVersion().SubtractMinor(1).String(),
utilversion.DefaultKubeEffectiveVersion().BinaryVersion().String(),
}
}
// GetEtcdStorageData returns etcd data for all persisted objects at the latest release version.
// It is exported so that it can be reused across multiple tests.
// It returns a new map on every invocation to prevent different tests from mutating shared state.
func GetEtcdStorageData() map[schema.GroupVersionResource]StorageData {
return GetEtcdStorageDataForNamespace("etcdstoragepathtestnamespace")
return GetEtcdStorageDataServedAt(utilversion.DefaultKubeBinaryVersion, false)
}
// GetEtcdStorageDataForNamespace returns etcd data for all persisted objects.
// GetEtcdStorageDataServedAt returns etcd data for all persisted objects at a particular release version.
// It is exported so that it can be reused across multiple tests.
// It returns a new map on every invocation to prevent different tests from mutating shared state.
func GetEtcdStorageDataServedAt(version string, removeAlphas bool) map[schema.GroupVersionResource]StorageData {
return GetEtcdStorageDataForNamespaceServedAt("etcdstoragepathtestnamespace", version, removeAlphas)
}
// GetEtcdStorageDataForNamespace returns etcd data for all persisted objects at the latest release version.
// It is exported so that it can be reused across multiple tests.
// It returns a new map on every invocation to prevent different tests from mutating shared state.
// Namespaced objects keys are computed for the specified namespace.
func GetEtcdStorageDataForNamespace(namespace string) map[schema.GroupVersionResource]StorageData {
return GetEtcdStorageDataForNamespaceServedAt(namespace, utilversion.DefaultKubeBinaryVersion, false)
}
// GetEtcdStorageDataForNamespaceServedAt returns etcd data for all persisted objects at a particular release version.
// It is exported so that it can be reused across multiple tests.
// It returns a new map on every invocation to prevent different tests from mutating shared state.
// Namespaced objects keys are computed for the specified namespace.
func GetEtcdStorageDataForNamespaceServedAt(namespace string, version string, removeAlphas bool) map[schema.GroupVersionResource]StorageData {
image := image.GetE2EImage(image.BusyBox)
etcdStorageData := map[schema.GroupVersionResource]StorageData{
// k8s.io/kubernetes/pkg/api/v1
@ -465,22 +493,46 @@ func GetEtcdStorageDataForNamespace(namespace string) map[schema.GroupVersionRes
},
// --
// k8s.io/kubernetes/pkg/apis/storage/v1
gvr("storage.k8s.io", "v1", "csinodes"): {
Stub: `{"metadata": {"name": "csini2"}, "spec": {"drivers": [{"name": "test-driver", "nodeID": "localhost", "topologyKeys": ["company.com/zone1", "company.com/zone2"]}]}}`,
ExpectedEtcdPath: "/registry/csinodes/csini2",
},
// --
// k8s.io/kubernetes/pkg/apis/storage/v1
gvr("storage.k8s.io", "v1", "csidrivers"): {
Stub: `{"metadata": {"name": "csid2"}, "spec": {"attachRequired": true, "podInfoOnMount": true}}`,
ExpectedEtcdPath: "/registry/csidrivers/csid2",
},
// --
}
// add csinodes
// k8s.io/kubernetes/pkg/apis/storage/v1
etcdStorageData[gvr("storage.k8s.io", "v1", "csinodes")] = StorageData{
Stub: `{"metadata": {"name": "csini2"}, "spec": {"drivers": [{"name": "test-driver", "nodeID": "localhost", "topologyKeys": ["company.com/zone1", "company.com/zone2"]}]}}`,
ExpectedEtcdPath: "/registry/csinodes/csini2",
// Delete types no longer served or not yet added at a particular emulated version.
// When adding a brand new type non-alpha type in the latest release, please ensure that
// it is removed in previous emulated versions otherwise emulated version tests will fail.
// TODO: derive this programatically from gvk --> instance --> APILifecycle info
switch version {
case "1.33":
delete(etcdStorageData, gvr("flowcontrol.apiserver.k8s.io", "v1beta3", "flowschemas"))
delete(etcdStorageData, gvr("flowcontrol.apiserver.k8s.io", "v1beta3", "prioritylevelconfigurations"))
case "1.32":
delete(etcdStorageData, gvr("flowcontrol.apiserver.k8s.io", "v1beta3", "flowschemas"))
delete(etcdStorageData, gvr("flowcontrol.apiserver.k8s.io", "v1beta3", "prioritylevelconfigurations"))
case "1.31":
delete(etcdStorageData, gvr("resource.k8s.io", "v1beta1", "deviceclasses"))
delete(etcdStorageData, gvr("resource.k8s.io", "v1beta1", "resourceclaims"))
delete(etcdStorageData, gvr("resource.k8s.io", "v1beta1", "resourceclaimtemplates"))
delete(etcdStorageData, gvr("resource.k8s.io", "v1beta1", "resourceslices"))
}
// add csidrivers
// k8s.io/kubernetes/pkg/apis/storage/v1
etcdStorageData[gvr("storage.k8s.io", "v1", "csidrivers")] = StorageData{
Stub: `{"metadata": {"name": "csid2"}, "spec": {"attachRequired": true, "podInfoOnMount": true}}`,
ExpectedEtcdPath: "/registry/csidrivers/csid2",
if removeAlphas {
for key := range etcdStorageData {
if strings.Contains(key.Version, "alpha") {
delete(etcdStorageData, key)
}
}
}
return etcdStorageData
}

41
test/integration/etcd/etcd_storage_path_test.go
View File

@ -39,9 +39,12 @@ import (
"k8s.io/apimachinery/pkg/runtime/serializer/recognizer"
utiljson "k8s.io/apimachinery/pkg/util/json"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/version"
"k8s.io/apiserver/pkg/util/feature"
"k8s.io/client-go/dynamic"
featuregatetesting "k8s.io/component-base/featuregate/testing"
componentbaseversion "k8s.io/component-base/version"
"k8s.io/kubernetes/cmd/kube-apiserver/app/options"
)
// Only add kinds to this list when this a virtual resource with get and create verbs that doesn't actually
@ -75,13 +78,32 @@ var allowMissingTestdataFixtures = map[schema.GroupVersionKind]bool{
// It will also fail when a type gets moved to a different location. Be very careful in this situation because
// it essentially means that you will be break old clusters unless you create some migration path for the old data.
func TestEtcdStoragePath(t *testing.T) {
// KUBE_APISERVER_SERVE_REMOVED_APIS_FOR_ONE_RELEASE allows for APIs pending removal to not block tests
t.Setenv("KUBE_APISERVER_SERVE_REMOVED_APIS_FOR_ONE_RELEASE", "true")
supportedVersions := GetSupportedEmulatedVersions()
for _, v := range supportedVersions {
t.Run(v, func(t *testing.T) {
testEtcdStoragePathWithVersion(t, v)
})
}
}
featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, "AllAlpha", true)
featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, "AllBeta", true)
func testEtcdStoragePathWithVersion(t *testing.T, v string) {
if v == componentbaseversion.DefaultKubeBinaryVersion {
featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, "AllAlpha", true)
featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, "AllBeta", true)
} else {
// Only test for beta and GA APIs with emulated version.
featuregatetesting.SetFeatureGateEmulationVersionDuringTest(t, feature.DefaultFeatureGate, version.MustParse(v))
featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, "AllBeta", true)
registerEffectiveEmulationVersion(t)
}
apiServer := StartRealAPIServerOrDie(t, func(opts *options.ServerRunOptions) {
// Disable alphas when emulating previous versions.
if v != componentbaseversion.DefaultKubeBinaryVersion {
opts.Options.APIEnablement.RuntimeConfig["api/alpha"] = "false"
}
})
apiServer := StartRealAPIServerOrDie(t)
defer apiServer.Cleanup()
defer dumpEtcdKVOnFailure(t, apiServer.KV)
@ -91,7 +113,14 @@ func TestEtcdStoragePath(t *testing.T) {
t.Fatal(err)
}
etcdStorageData := GetEtcdStorageData()
var etcdStorageData map[schema.GroupVersionResource]StorageData
if v == componentbaseversion.DefaultKubeBinaryVersion {
etcdStorageData = GetEtcdStorageDataForNamespaceServedAt("etcdstoragepathtestnamespace", v, false)
} else {
// Drop alphas from etcd data fixtures when emulating previous versions
// as alphas are not supported with emulation.
etcdStorageData = GetEtcdStorageDataForNamespaceServedAt("etcdstoragepathtestnamespace", v, true)
}
kindSeen := sets.NewString()
pathSeen := map[string][]schema.GroupVersionResource{}

19
test/integration/etcd/server.go
View File

@ -36,14 +36,19 @@ import (
"k8s.io/apimachinery/pkg/runtime/schema"
utilerrors "k8s.io/apimachinery/pkg/util/errors"
"k8s.io/apimachinery/pkg/util/json"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/wait"
genericapiserveroptions "k8s.io/apiserver/pkg/server/options"
"k8s.io/apiserver/pkg/util/feature"
cacheddiscovery "k8s.io/client-go/discovery/cached/memory"
"k8s.io/client-go/dynamic"
clientset "k8s.io/client-go/kubernetes"
restclient "k8s.io/client-go/rest"
"k8s.io/client-go/restmapper"
utiltesting "k8s.io/client-go/util/testing"
"k8s.io/component-base/featuregate"
featuregatetesting "k8s.io/component-base/featuregate/testing"
utilversion "k8s.io/component-base/version"
"k8s.io/kubernetes/cmd/kube-apiserver/app"
"k8s.io/kubernetes/cmd/kube-apiserver/app/options"
"k8s.io/kubernetes/test/integration"
@ -62,11 +67,23 @@ AwEHoUQDQgAEH6cuzP8XuD5wal6wf9M6xDljTOPLX2i8uIp/C/ASqiIGUeeKQtX0
/IR3qCXyThP/dbCiHrF3v1cuhBOHY8CLVg==
-----END EC PRIVATE KEY-----`
func registerEffectiveEmulationVersion(t *testing.T) {
featureGate := feature.DefaultMutableFeatureGate
featureGate.AddMetrics()
effectiveVersion := utilversion.DefaultKubeEffectiveVersion()
effectiveVersion.SetEmulationVersion(featureGate.EmulationVersion())
featuregatetesting.SetFeatureGateEmulationVersionDuringTest(t, featureGate, effectiveVersion.EmulationVersion())
featuregate.DefaultComponentGlobalsRegistry.Reset()
utilruntime.Must(featuregate.DefaultComponentGlobalsRegistry.Register(featuregate.DefaultKubeComponent, effectiveVersion, featureGate))
}
// StartRealAPIServerOrDie starts an API server that is appropriate for use in tests that require one of every resource
func StartRealAPIServerOrDie(t *testing.T, configFuncs ...func(*options.ServerRunOptions)) *APIServer {
tCtx := ktesting.Init(t)
certDir, err := os.MkdirTemp("", t.Name())
// Strip out "/" in subtests
certDir, err := os.MkdirTemp("", strings.ReplaceAll(t.Name(), "/", ""))
if err != nil {
t.Fatal(err)
}