github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2026-01-29 21:29:24 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #55548 from dhilipkumars/Extend-testapiserver

Browse Source 
Automatic merge from submit-queue (batch tested with PRs 55545, 55548, 55815, 56136, 56185). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Extend apiserver testserver

**What this PR does / why we need it**:
Extend the test pkg of api-server a little bit so that it can be used by others
**Which issue(s) this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close the issue(s) when PR gets merged)*:
Fixes #

**Special notes for your reviewer**:
We wanted to use this test pkg while creating a new integration test fixture, the idea of starting the api-server without actually starting a proper etcd (integration etcd ) is very useful, if would be great to extend this for other usecases such as `insecure` mode too.

It will also be useful to get the working `tmpdir` returned so that we could store additional certs (such as service account public and private keys and kubeconf for this apiserver) if needed, which will be cleaned up automatically by the `teardownFn` when the api server is terminating.

**Release note**:

```release-note
NONE
```
ref: https://github.com/kubernetes/kubernetes/pull/50144
This commit is contained in:
Kubernetes Submit Queue
2017-11-22 16:52:26 -08:00
committed by GitHub
parent 11cf604377 fd8758b047
commit 704a0b52b0
14 changed files with 284 additions and 221 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
cmd/kube-apiserver/app/testing/BUILD
View File

@@ -3,32 +3,6 @@ package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
"go_test",
)
go_test(
name = "go_default_test",
srcs = ["server_test.go"],
importpath = "k8s.io/kubernetes/cmd/kube-apiserver/app/testing",
library = ":go_default_library",
deps = [
"//vendor/k8s.io/api/admissionregistration/v1alpha1:go_default_library",
"//vendor/k8s.io/api/apps/v1beta1:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library",
"//vendor/k8s.io/api/networking/v1:go_default_library",
"//vendor/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1:go_default_library",
"//vendor/k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1/unstructured:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
"//vendor/k8s.io/apiserver/pkg/features:go_default_library",
"//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library",
"//vendor/k8s.io/apiserver/pkg/util/feature/testing:go_default_library",
"//vendor/k8s.io/client-go/dynamic:go_default_library",
"//vendor/k8s.io/client-go/kubernetes:go_default_library",
],
)
go_library(
@@ -38,9 +12,10 @@ go_library(
deps = [
"//cmd/kube-apiserver/app:go_default_library",
"//cmd/kube-apiserver/app/options:go_default_library",
"//vendor/github.com/spf13/pflag:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
"//vendor/k8s.io/apiserver/pkg/registry/generic/registry:go_default_library",
"//vendor/k8s.io/apiserver/pkg/storage/etcd/testing:go_default_library",
"//vendor/k8s.io/apiserver/pkg/storage/storagebackend:go_default_library",
"//vendor/k8s.io/client-go/kubernetes:go_default_library",
"//vendor/k8s.io/client-go/rest:go_default_library",
],

391
cmd/kube-apiserver/app/testing/server_test.go
View File

@@ -1,391 +0,0 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package testing
import (
"encoding/json"
"fmt"
"testing"
"time"
admissionregistrationv1alpha1 "k8s.io/api/admissionregistration/v1alpha1"
appsv1beta1 "k8s.io/api/apps/v1beta1"
corev1 "k8s.io/api/core/v1"
networkingv1 "k8s.io/api/networking/v1"
apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/apiserver/pkg/features"
utilfeature "k8s.io/apiserver/pkg/util/feature"
utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing"
"k8s.io/client-go/dynamic"
"k8s.io/client-go/kubernetes"
)
func TestRun(t *testing.T) {
config, tearDown := StartTestServerOrDie(t)
defer tearDown()
client, err := kubernetes.NewForConfig(config)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
// test whether the server is really healthy after /healthz told us so
t.Logf("Creating Deployment directly after being healthy")
var replicas int32 = 1
_, err = client.AppsV1beta1().Deployments("default").Create(&appsv1beta1.Deployment{
TypeMeta: metav1.TypeMeta{
Kind: "Deployment",
APIVersion: "apps/v1beta1",
},
ObjectMeta: metav1.ObjectMeta{
Namespace: "default",
Name: "test",
},
Spec: appsv1beta1.DeploymentSpec{
Replicas: &replicas,
Strategy: appsv1beta1.DeploymentStrategy{
Type: appsv1beta1.RollingUpdateDeploymentStrategyType,
},
Template: corev1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Labels: map[string]string{"foo": "bar"},
},
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "foo",
Image: "foo",
},
},
},
},
},
})
if err != nil {
t.Fatalf("Failed to create deployment: %v", err)
}
}
func TestCRDShadowGroup(t *testing.T) {
config, tearDown := StartTestServerOrDie(t)
defer tearDown()
kubeclient, err := kubernetes.NewForConfig(config)
if err != nil {
t.Fatalf("Unexpected error: %v", err)
}
apiextensionsclient, err := apiextensionsclientset.NewForConfig(config)
if err != nil {
t.Fatalf("Unexpected error: %v", err)
}
t.Logf("Creating a NetworkPolicy")
nwPolicy, err := kubeclient.NetworkingV1().NetworkPolicies("default").Create(&networkingv1.NetworkPolicy{
ObjectMeta: metav1.ObjectMeta{Name: "abc", Namespace: metav1.NamespaceDefault},
Spec: networkingv1.NetworkPolicySpec{
PodSelector: metav1.LabelSelector{MatchLabels: map[string]string{"foo": "bar"}},
Ingress: []networkingv1.NetworkPolicyIngressRule{},
},
})
if err != nil {
t.Fatalf("Failed to create NetworkPolicy: %v", err)
}
t.Logf("Trying to shadow networking group")
crd := &apiextensionsv1beta1.CustomResourceDefinition{
ObjectMeta: metav1.ObjectMeta{
Name: "foos." + networkingv1.GroupName,
},
Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{
Group: networkingv1.GroupName,
Version: networkingv1.SchemeGroupVersion.Version,
Scope: apiextensionsv1beta1.ClusterScoped,
Names: apiextensionsv1beta1.CustomResourceDefinitionNames{
Plural: "foos",
Kind: "Foo",
},
},
}
if _, err = apiextensionsclient.ApiextensionsV1beta1().CustomResourceDefinitions().Create(crd); err != nil {
t.Fatalf("Failed to create networking group CRD: %v", err)
}
if err := waitForEstablishedCRD(apiextensionsclient, crd.Name); err != nil {
t.Fatalf("Failed to establish networking group CRD: %v", err)
}
// wait to give aggregator time to update
time.Sleep(2 * time.Second)
t.Logf("Checking that we still see the NetworkPolicy")
_, err = kubeclient.NetworkingV1().NetworkPolicies(nwPolicy.Namespace).Get(nwPolicy.Name, metav1.GetOptions{})
if err != nil {
t.Errorf("Failed to get NetworkPolocy: %v", err)
}
t.Logf("Checking that crd resource does not show up in networking group")
found, err := crdExistsInDiscovery(apiextensionsclient, crd)
if err != nil {
t.Fatalf("unexpected discovery error: %v", err)
}
if found {
t.Errorf("CRD resource shows up in discovery, but shouldn't.")
}
}
func TestCRD(t *testing.T) {
defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.Initializers, true)()
config, tearDown := StartTestServerOrDie(t)
defer tearDown()
kubeclient, err := kubernetes.NewForConfig(config)
if err != nil {
t.Fatalf("Unexpected error: %v", err)
}
apiextensionsclient, err := apiextensionsclientset.NewForConfig(config)
if err != nil {
t.Fatalf("Unexpected error: %v", err)
}
t.Logf("Trying to create a custom resource without conflict")
crd := &apiextensionsv1beta1.CustomResourceDefinition{
ObjectMeta: metav1.ObjectMeta{
Name: "foos.cr.bar.com",
},
Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{
Group: "cr.bar.com",
Version: "v1",
Scope: apiextensionsv1beta1.NamespaceScoped,
Names: apiextensionsv1beta1.CustomResourceDefinitionNames{
Plural: "foos",
Kind: "Foo",
},
},
}
if _, err = apiextensionsclient.ApiextensionsV1beta1().CustomResourceDefinitions().Create(crd); err != nil {
t.Fatalf("Failed to create foos.cr.bar.com CRD; %v", err)
}
if err := waitForEstablishedCRD(apiextensionsclient, crd.Name); err != nil {
t.Fatalf("Failed to establish foos.cr.bar.com CRD: %v", err)
}
if err := wait.PollImmediate(500*time.Millisecond, 30*time.Second, func() (bool, error) {
return crdExistsInDiscovery(apiextensionsclient, crd)
}); err != nil {
t.Fatalf("Failed to see foos.cr.bar.com in discovery: %v", err)
}
t.Logf("Trying to access foos.cr.bar.com with dynamic client")
barComConfig := *config
barComConfig.GroupVersion = &schema.GroupVersion{Group: "cr.bar.com", Version: "v1"}
barComConfig.APIPath = "/apis"
barComClient, err := dynamic.NewClient(&barComConfig)
if err != nil {
t.Fatalf("Unexpected error: %v", err)
}
_, err = barComClient.Resource(&metav1.APIResource{Name: "foos", Namespaced: true}, "default").List(metav1.ListOptions{})
if err != nil {
t.Errorf("Failed to list foos.cr.bar.com instances: %v", err)
}
t.Logf("Creating InitializerConfiguration")
_, err = kubeclient.AdmissionregistrationV1alpha1().InitializerConfigurations().Create(&admissionregistrationv1alpha1.InitializerConfiguration{
ObjectMeta: metav1.ObjectMeta{
Name: "foos.cr.bar.com",
},
Initializers: []admissionregistrationv1alpha1.Initializer{
{
Name: "cr.bar.com",
Rules: []admissionregistrationv1alpha1.Rule{
{
APIGroups: []string{"cr.bar.com"},
APIVersions: []string{"*"},
Resources: []string{"*"},
},
},
},
},
})
if err != nil {
t.Fatalf("Failed to create InitializerConfiguration: %v", err)
}
// TODO DO NOT MERGE THIS
time.Sleep(5 * time.Second)
t.Logf("Creating Foo instance")
foo := &Foo{
TypeMeta: metav1.TypeMeta{
APIVersion: "cr.bar.com/v1",
Kind: "Foo",
},
ObjectMeta: metav1.ObjectMeta{Name: "foo"},
}
unstructuredFoo, err := unstructuredFoo(foo)
if err != nil {
t.Fatalf("Unable to create Foo: %v", err)
}
createErr := make(chan error, 1)
go func() {
_, err := barComClient.Resource(&metav1.APIResource{Name: "foos", Namespaced: true}, "default").Create(unstructuredFoo)
t.Logf("Foo instance create returned: %v", err)
if err != nil {
createErr <- err
}
}()
err = wait.PollImmediate(100*time.Millisecond, wait.ForeverTestTimeout, func() (bool, error) {
select {
case createErr := <-createErr:
return true, createErr
default:
}
t.Logf("Checking that Foo instance is visible with IncludeUninitialized=true")
_, err := barComClient.Resource(&metav1.APIResource{Name: "foos", Namespaced: true}, "default").Get(foo.ObjectMeta.Name, metav1.GetOptions{
IncludeUninitialized: true,
})
switch {
case err == nil:
return true, nil
case errors.IsNotFound(err):
return false, nil
default:
return false, err
}
})
if err != nil {
t.Fatalf("Unexpected error: %v", err)
}
t.Logf("Removing initializer from Foo instance")
success := false
for i := 0; i < 10; i++ {
// would love to replace the following with a patch, but removing strings from the intitializer array
// is not what JSON (Merge) patch authors had in mind.
fooUnstructured, err := barComClient.Resource(&metav1.APIResource{Name: "foos", Namespaced: true}, "default").Get(foo.ObjectMeta.Name, metav1.GetOptions{
IncludeUninitialized: true,
})
if err != nil {
t.Fatalf("Error getting Foo instance: %v", err)
}
bs, _ := fooUnstructured.MarshalJSON()
t.Logf("Got Foo instance: %v", string(bs))
foo := Foo{}
if err := json.Unmarshal(bs, &foo); err != nil {
t.Fatalf("Error parsing Foo instance: %v", err)
}
// remove initialize
if foo.ObjectMeta.Initializers == nil {
t.Fatalf("Expected initializers to be set in Foo instance")
}
found := false
for i := range foo.ObjectMeta.Initializers.Pending {
if foo.ObjectMeta.Initializers.Pending[i].Name == "cr.bar.com" {
foo.ObjectMeta.Initializers.Pending = append(foo.ObjectMeta.Initializers.Pending[:i], foo.ObjectMeta.Initializers.Pending[i+1:]...)
found = true
break
}
}
if !found {
t.Fatalf("Expected cr.bar.com as initializer on Foo instance")
}
if len(foo.ObjectMeta.Initializers.Pending) == 0 && foo.ObjectMeta.Initializers.Result == nil {
foo.ObjectMeta.Initializers = nil
}
bs, err = json.Marshal(&foo)
if err != nil {
t.Fatalf("Unexpected error: %v", err)
}
fooUnstructured.UnmarshalJSON(bs)
_, err = barComClient.Resource(&metav1.APIResource{Name: "foos", Namespaced: true}, "default").Update(fooUnstructured)
if err != nil && !errors.IsConflict(err) {
t.Fatalf("Failed to update Foo instance: %v", err)
} else if err == nil {
success = true
break
}
}
if !success {
t.Fatalf("Failed to remove initializer from Foo object")
}
t.Logf("Checking that Foo instance is visible after removing the initializer")
if _, err := barComClient.Resource(&metav1.APIResource{Name: "foos", Namespaced: true}, "default").Get(foo.ObjectMeta.Name, metav1.GetOptions{}); err != nil {
t.Errorf("Unexpected error: %v", err)
}
}
type Foo struct {
metav1.TypeMeta `json:",inline"`
metav1.ObjectMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"`
}
func unstructuredFoo(foo *Foo) (*unstructured.Unstructured, error) {
bs, err := json.Marshal(foo)
if err != nil {
return nil, err
}
ret := &unstructured.Unstructured{}
if err = ret.UnmarshalJSON(bs); err != nil {
return nil, err
}
return ret, nil
}
func waitForEstablishedCRD(client apiextensionsclientset.Interface, name string) error {
return wait.PollImmediate(500*time.Millisecond, wait.ForeverTestTimeout, func() (bool, error) {
crd, err := client.ApiextensionsV1beta1().CustomResourceDefinitions().Get(name, metav1.GetOptions{})
if err != nil {
return false, err
}
for _, cond := range crd.Status.Conditions {
switch cond.Type {
case apiextensionsv1beta1.Established:
if cond.Status == apiextensionsv1beta1.ConditionTrue {
return true, err
}
case apiextensionsv1beta1.NamesAccepted:
if cond.Status == apiextensionsv1beta1.ConditionFalse {
fmt.Printf("Name conflict: %v\n", cond.Reason)
}
}
}
return false, nil
})
}
func crdExistsInDiscovery(client apiextensionsclientset.Interface, crd *apiextensionsv1beta1.CustomResourceDefinition) (bool, error) {
resourceList, err := client.Discovery().ServerResourcesForGroupVersion(crd.Spec.Group + "/" + crd.Spec.Version)
if err != nil {
return false, nil
}
for _, resource := range resourceList.APIResources {
if resource.Name == crd.Spec.Names.Plural {
return true, nil
}
}
return false, nil
}

79
cmd/kube-apiserver/app/testing/testserver.go
View File

@@ -21,13 +21,14 @@ import (
"io/ioutil"
"net"
"os"
"strings"
"testing"
"time"
pflag "github.com/spf13/pflag"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/apiserver/pkg/registry/generic/registry"
etcdtesting "k8s.io/apiserver/pkg/storage/etcd/testing"
"k8s.io/apiserver/pkg/storage/storagebackend"
"k8s.io/client-go/kubernetes"
restclient "k8s.io/client-go/rest"
"k8s.io/kubernetes/cmd/kube-apiserver/app"
@@ -37,15 +38,21 @@ import (
// TearDownFunc is to be called to tear down a test server.
type TearDownFunc func()
// StartTestServer starts a etcd server and kube-apiserver. A rest client config and a tear-down func
// are returned.
// TestServer return values supplied by kube-test-ApiServer
type TestServer struct {
ClientConfig *restclient.Config // Rest client config
ServerOpts *options.ServerRunOptions // ServerOpts
TearDownFn TearDownFunc // TearDown function
TmpDir string // Temp Dir used, by the apiserver
}
// StartTestServer starts a etcd server and kube-apiserver. A rest client config and a tear-down func,
// and location of the tmpdir are returned.
//
// Note: we return a tear-down func instead of a stop channel because the later will leak temporariy
// files that becaues Golang testing's call to os.Exit will not give a stop channel go routine
// enough time to remove temporariy files.
func StartTestServer(t *testing.T) (result *restclient.Config, tearDownForCaller TearDownFunc, err error) {
var tmpDir string
var etcdServer *etcdtesting.EtcdTestServer
func StartTestServer(t *testing.T, customFlags []string, storageConfig *storagebackend.Config) (result TestServer, err error) {
// TODO : Remove TrackStorageCleanup below when PR
// https://github.com/kubernetes/kubernetes/pull/50690
@@ -56,46 +63,45 @@ func StartTestServer(t *testing.T) (result *restclient.Config, tearDownForCaller
tearDown := func() {
registry.CleanupStorage()
close(stopCh)
if etcdServer != nil {
etcdServer.Terminate(t)
}
if len(tmpDir) != 0 {
os.RemoveAll(tmpDir)
if len(result.TmpDir) != 0 {
os.RemoveAll(result.TmpDir)
}
}
defer func() {
if tearDownForCaller == nil {
if result.TearDownFn == nil {
tearDown()
}
}()
t.Logf("Starting etcd...")
etcdServer, storageConfig := etcdtesting.NewUnsecuredEtcd3TestClientServer(t)
tmpDir, err = ioutil.TempDir("", "kubernetes-kube-apiserver")
result.TmpDir, err = ioutil.TempDir("", "kubernetes-kube-apiserver")
if err != nil {
return nil, nil, fmt.Errorf("failed to create temp dir: %v", err)
return result, fmt.Errorf("failed to create temp dir: %v", err)
}
fs := pflag.NewFlagSet("test", pflag.PanicOnError)
s := options.NewServerRunOptions()
s.AddFlags(fs)
s.InsecureServing.BindPort = 0
s.SecureServing.Listener, s.SecureServing.BindPort, err = createListenerOnFreePort()
if err != nil {
return nil, nil, fmt.Errorf("failed to create listener: %v", err)
return result, fmt.Errorf("failed to create listener: %v", err)
}
s.SecureServing.ServerCert.CertDirectory = tmpDir
s.SecureServing.ServerCert.CertDirectory = result.TmpDir
s.ServiceClusterIPRange.IP = net.IPv4(10, 0, 0, 0)
s.ServiceClusterIPRange.Mask = net.CIDRMask(16, 32)
s.Etcd.StorageConfig = *storageConfig
s.Etcd.DefaultStorageMediaType = "application/json"
s.Admission.PluginNames = strings.Split("Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds", ",")
s.APIEnablement.RuntimeConfig.Set("api/all=true")
fs.Parse(customFlags)
t.Logf("Starting kube-apiserver on port %d...", s.SecureServing.BindPort)
server, err := app.CreateServerChain(s, stopCh)
if err != nil {
return nil, nil, fmt.Errorf("failed to create server chain: %v", err)
return result, fmt.Errorf("failed to create server chain: %v", err)
}
go func(stopCh <-chan struct{}) {
if err := server.PrepareRun().Run(stopCh); err != nil {
@@ -104,9 +110,10 @@ func StartTestServer(t *testing.T) (result *restclient.Config, tearDownForCaller
}(stopCh)
t.Logf("Waiting for /healthz to be ok...")
client, err := kubernetes.NewForConfig(server.LoopbackClientConfig)
if err != nil {
return nil, nil, fmt.Errorf("failed to create a client: %v", err)
return result, fmt.Errorf("failed to create a client: %v", err)
}
err = wait.Poll(100*time.Millisecond, 30*time.Second, func() (bool, error) {
result := client.CoreV1().RESTClient().Get().AbsPath("/healthz").Do()
@@ -118,23 +125,27 @@ func StartTestServer(t *testing.T) (result *restclient.Config, tearDownForCaller
return false, nil
})
if err != nil {
return nil, nil, fmt.Errorf("failed to wait for /healthz to return ok: %v", err)
return result, fmt.Errorf("failed to wait for /healthz to return ok: %v", err)
}
// from here the caller must call tearDown
return server.LoopbackClientConfig, tearDown, nil
result.ClientConfig = server.LoopbackClientConfig
result.ServerOpts = s
result.TearDownFn = tearDown
return result, nil
}
// StartTestServerOrDie calls StartTestServer with up to 5 retries on bind error and dies with
// t.Fatal if it does not succeed.
func StartTestServerOrDie(t *testing.T) (*restclient.Config, TearDownFunc) {
config, td, err := StartTestServer(t)
// StartTestServerOrDie calls StartTestServer t.Fatal if it does not succeed.
func StartTestServerOrDie(t *testing.T, flags []string, storageConfig *storagebackend.Config) *TestServer {
result, err := StartTestServer(t, flags, storageConfig)
if err == nil {
return config, td
return &result
}
t.Fatalf("Failed to launch server: %v", err)
return nil, nil
t.Fatalf("failed to launch server: %v", err)
return nil
}
func createListenerOnFreePort() (net.Listener, int, error) {