storage version integration test: check the test server's health before running

we disabled the /healthz check because our test blocks one post-start
hook from finishing. Instead we should check all the other /healthz/...
endpoints before running the tests

cmd/kube-apiserver/app/testing/BUILD

@@ -22,6 +22,7 @@ go_library(
         "//staging/src/k8s.io/client-go/kubernetes:go_default_library",
         "//staging/src/k8s.io/client-go/rest:go_default_library",
         "//staging/src/k8s.io/client-go/util/cert:go_default_library",
+        "//staging/src/k8s.io/kube-aggregator/pkg/apiserver:go_default_library",
         "//test/utils:go_default_library",
         "//vendor/github.com/spf13/pflag:go_default_library",
     ],

cmd/kube-apiserver/app/testing/testserver.go

@@ -38,6 +38,7 @@ import (
 	"k8s.io/client-go/kubernetes"
 	restclient "k8s.io/client-go/rest"
 	"k8s.io/client-go/util/cert"
+	"k8s.io/kube-aggregator/pkg/apiserver"
 	"k8s.io/kubernetes/cmd/kube-apiserver/app"
 	"k8s.io/kubernetes/cmd/kube-apiserver/app/options"
 	testutil "k8s.io/kubernetes/test/utils"
@@ -201,54 +202,60 @@ func StartTestServer(t Logger, instanceOptions *TestServerInstanceOptions, custo
 		}
 	}(stopCh)
 
-	// skip healthz check when we test the storage version manager poststart hook
+	t.Logf("Waiting for /healthz to be ok...")
-	if instanceOptions.StorageVersionWrapFunc == nil {
-		t.Logf("Waiting for /healthz to be ok...")
 
-		client, err := kubernetes.NewForConfig(server.GenericAPIServer.LoopbackClientConfig)
+	client, err := kubernetes.NewForConfig(server.GenericAPIServer.LoopbackClientConfig)
-		if err != nil {
+	if err != nil {
-			return result, fmt.Errorf("failed to create a client: %v", err)
+		return result, fmt.Errorf("failed to create a client: %v", err)
+	}
+
+	// wait until healthz endpoint returns ok
+	err = wait.Poll(100*time.Millisecond, time.Minute, func() (bool, error) {
+		select {
+		case err := <-errCh:
+			return false, err
+		default:
 		}
 
-		// wait until healthz endpoint returns ok
+		req := client.CoreV1().RESTClient().Get().AbsPath("/healthz")
-		err = wait.Poll(100*time.Millisecond, time.Minute, func() (bool, error) {
+		// The storage version bootstrap test wraps the storage version post-start
-			select {
+		// hook, so the hook won't become health when the server bootstraps
-			case err := <-errCh:
+		if instanceOptions.StorageVersionWrapFunc != nil {
-				return false, err
+			// We hardcode the param instead of having a new instanceOptions field
-			default:
+			// to avoid confusing users with more options.
-			}
+			storageVersionCheck := fmt.Sprintf("poststarthook/%s", apiserver.StorageVersionPostStartHookName)
+			req.Param("exclude", storageVersionCheck)
+		}
+		result := req.Do(context.TODO())
+		status := 0
+		result.StatusCode(&status)
+		if status == 200 {
+			return true, nil
+		}
+		return false, nil
+	})
+	if err != nil {
+		return result, fmt.Errorf("failed to wait for /healthz to return ok: %v", err)
+	}
 
-			result := client.CoreV1().RESTClient().Get().AbsPath("/healthz").Do(context.TODO())
+	// wait until default namespace is created
-			status := 0
+	err = wait.Poll(100*time.Millisecond, 30*time.Second, func() (bool, error) {
-			result.StatusCode(&status)
+		select {
-			if status == 200 {
+		case err := <-errCh:
-				return true, nil
+			return false, err
+		default:
+		}
+
+		if _, err := client.CoreV1().Namespaces().Get(context.TODO(), "default", metav1.GetOptions{}); err != nil {
+			if !errors.IsNotFound(err) {
+				t.Logf("Unable to get default namespace: %v", err)
 			}
 			return false, nil
-		})
-		if err != nil {
-			return result, fmt.Errorf("failed to wait for /healthz to return ok: %v", err)
-		}
-
-		// wait until default namespace is created
-		err = wait.Poll(100*time.Millisecond, 30*time.Second, func() (bool, error) {
-			select {
-			case err := <-errCh:
-				return false, err
-			default:
-			}
-
-			if _, err := client.CoreV1().Namespaces().Get(context.TODO(), "default", metav1.GetOptions{}); err != nil {
-				if !errors.IsNotFound(err) {
-					t.Logf("Unable to get default namespace: %v", err)
-				}
-				return false, nil
-			}
-			return true, nil
-		})
-		if err != nil {
-			return result, fmt.Errorf("failed to wait for default namespace to be created: %v", err)
 		}
+		return true, nil
+	})
+	if err != nil {
+		return result, fmt.Errorf("failed to wait for default namespace to be created: %v", err)
 	}
 
 	// from here the caller must call tearDown

staging/src/k8s.io/kube-aggregator/pkg/apiserver/apiserver.go

@@ -64,8 +64,12 @@ func init() {
 	)
 }
 
-// legacyAPIServiceName is the fixed name of the only non-groupified API version
+const (
-const legacyAPIServiceName = "v1."
+	// legacyAPIServiceName is the fixed name of the only non-groupified API version
+	legacyAPIServiceName = "v1."
+	// StorageVersionPostStartHookName is the name of the storage version updater post start hook.
+	StorageVersionPostStartHookName = "built-in-resources-storage-version-updater"
+)
 
 // ExtraConfig represents APIServices-specific configuration
 type ExtraConfig struct {
@@ -269,7 +273,7 @@ func (c completedConfig) NewWithDelegate(delegationTarget genericapiserver.Deleg
 		utilfeature.DefaultFeatureGate.Enabled(genericfeatures.APIServerIdentity) {
 		// Spawn a goroutine in aggregator apiserver to update storage version for
 		// all built-in resources
-		s.GenericAPIServer.AddPostStartHookOrDie("built-in-resources-storage-version-updater", func(hookContext genericapiserver.PostStartHookContext) error {
+		s.GenericAPIServer.AddPostStartHookOrDie(StorageVersionPostStartHookName, func(hookContext genericapiserver.PostStartHookContext) error {
 			// Wait for apiserver-identity to exist first before updating storage
 			// versions, to avoid storage version GC accidentally garbage-collecting
 			// storage versions.