Merge pull request #30211 from Random-Liu/statically-link-apiserver

Automatic merge from submit-queue

Node Conformance Test: Statically link apiserver

For #30122, #30174.
Based on #30116, #30198, #30200.

**Please only review the 4th PR.**

This PR is part of our roadmap to package node conformance test.
The 1st commit is from #30116, which started e2e services in a separate process.
The 2nd commit is from #30198, it statically linked etcd into the node e2e framework.
The 3rd commit is from #30200, it moved namespace controller into e2e services.

The 4th commit is new, it statically linked apiserver into the node e2e framework.

@dchen1107 @vishh 
/cc @kubernetes/sig-node @kubernetes/sig-testing

test/e2e_node/apiserver.go

@@ -0,0 +1,85 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e_node
+
+import (
+	"fmt"
+	"net"
+
+	apiserver "k8s.io/kubernetes/cmd/kube-apiserver/app"
+	"k8s.io/kubernetes/cmd/kube-apiserver/app/options"
+)
+
+const (
+	clusterIPRange          = "10.0.0.1/24"
+	apiserverClientURL      = "http://localhost:8080"
+	apiserverHealthCheckURL = apiserverClientURL + "/healthz"
+)
+
+// APIServer is a server which manages apiserver.
+type APIServer struct{}
+
+// NewAPIServer creates an apiserver.
+func NewAPIServer() *APIServer {
+	return &APIServer{}
+}
+
+// Start starts the apiserver, returns when apiserver is ready.
+func (a *APIServer) Start() error {
+	config := options.NewAPIServer()
+	config.StorageConfig.ServerList = []string{getEtcdClientURL()}
+	_, ipnet, err := net.ParseCIDR(clusterIPRange)
+	if err != nil {
+		return err
+	}
+	config.ServiceClusterIPRange = *ipnet
+	config.AllowPrivileged = true
+	errCh := make(chan error)
+	go func() {
+		defer close(errCh)
+		err := apiserver.Run(config)
+		if err != nil {
+			errCh <- fmt.Errorf("run apiserver error: %v", err)
+		}
+	}()
+
+	err = readinessCheck([]string{apiserverHealthCheckURL}, errCh)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// Stop stops the apiserver. Currently, there is no way to stop the apiserver.
+// The function is here only for completion.
+func (a *APIServer) Stop() error {
+	return nil
+}
+
+const apiserverName = "apiserver"
+
+func (a *APIServer) Name() string {
+	return apiserverName
+}
+
+func getAPIServerClientURL() string {
+	return apiserverClientURL
+}
+
+func getAPIServerHealthCheckURL() string {
+	return apiserverHealthCheckURL
+}

test/e2e_node/e2e_service.go

@@ -120,15 +120,13 @@ func RunE2EServices() {
 
 // Ports of different e2e services.
 const (
-	apiserverPort       = "8080"
 	kubeletPort         = "10250"
 	kubeletReadOnlyPort = "10255"
 )
 
 // Health check urls of different e2e services.
 var (
-	apiserverHealthCheckURL = getEndpoint(apiserverPort) + "/healthz"
+	kubeletHealthCheckURL = getEndpoint(kubeletReadOnlyPort) + "/healthz"
-	kubeletHealthCheckURL   = getEndpoint(kubeletReadOnlyPort) + "/healthz"
 )
 
 // getEndpoint generates endpoint url from service port.
@@ -139,7 +137,7 @@ func getEndpoint(port string) string {
 func getHealthCheckURLs() []string {
 	return []string{
 		getEtcdHealthCheckURL(),
-		apiserverHealthCheckURL,
+		getAPIServerHealthCheckURL(),
 		kubeletHealthCheckURL,
 	}
 }
@@ -152,6 +150,7 @@ type e2eService struct {
 
 	// All statically linked e2e services
 	etcdServer   *EtcdServer
+	apiServer    *APIServer
 	nsController *NamespaceController
 }
 
@@ -200,22 +199,18 @@ func (es *e2eService) start() error {
 	if _, err := getK8sBin("kubelet"); err != nil {
 		return err
 	}
-	if _, err := getK8sBin("kube-apiserver"); err != nil {
-		return err
-	}
 
 	err := es.startEtcd()
 	if err != nil {
 		return err
 	}
 
-	s, err := es.startApiServer()
+	err = es.startApiServer()
 	if err != nil {
 		return err
 	}
-	es.services = append(es.services, s)
 
-	s, err = es.startKubeletServer()
+	s, err := es.startKubeletServer()
 	if err != nil {
 		return err
 	}
@@ -293,6 +288,12 @@ func (es *e2eService) stop() {
 			glog.Errorf("Failed to stop %q: %v", es.nsController.Name(), err)
 		}
 	}
+	// Stop apiserver
+	if es.apiServer != nil {
+		if err := es.apiServer.Stop(); err != nil {
+			glog.Errorf("Failed to stop %q: %v", es.apiServer.Name(), err)
+		}
+	}
 	for _, s := range es.services {
 		if err := s.kill(); err != nil {
 			glog.Errorf("Failed to stop %v: %v", s.name, err)
@@ -323,22 +324,9 @@ func (es *e2eService) startEtcd() error {
 	return es.etcdServer.Start()
 }
 
-func (es *e2eService) startApiServer() (*server, error) {
+func (es *e2eService) startApiServer() error {
-	cmd := exec.Command("sudo", getApiServerBin(),
+	es.apiServer = NewAPIServer()
-		"--etcd-servers", getEtcdClientURL(),
+	return es.apiServer.Start()
-		"--insecure-bind-address", "0.0.0.0",
-		"--service-cluster-ip-range", "10.0.0.1/24",
-		"--kubelet-port", kubeletPort,
-		"--allow-privileged", "true",
-		"--v", LOG_VERBOSITY_LEVEL, "--logtostderr",
-	)
-	server := newServer(
-		"apiserver",
-		cmd,
-		nil,
-		[]string{apiserverHealthCheckURL},
-		"kube-apiserver.log")
-	return server, server.start()
 }
 
 func (es *e2eService) startNamespaceController() error {
@@ -370,7 +358,7 @@ func (es *e2eService) startKubeletServer() (*server, error) {
 		)
 	}
 	cmdArgs = append(cmdArgs,
-		"--api-servers", getEndpoint(apiserverPort),
+		"--api-servers", getAPIServerClientURL(),
 		"--address", "0.0.0.0",
 		"--port", kubeletPort,
 		"--read-only-port", kubeletReadOnlyPort,