Modify integration test to fill CCM test gap

cmd/cloud-controller-manager/main.go

@@ -31,6 +31,7 @@ import (
 
 	"github.com/spf13/pflag"
 
+	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/cloud-provider"
 	"k8s.io/cloud-provider/app"
 	cloudcontrollerconfig "k8s.io/cloud-provider/app/config"
@@ -44,11 +45,6 @@ import (
 	// e.g. _"k8s.io/legacy-cloud-providers/<provider>"
 )
 
-const (
-	// cloudProviderName shows an sample of using hard coded parameter, please edit the value for your case.
-	cloudProviderName = "SampleCloudProviderName"
-)
-
 func main() {
 	rand.Seed(time.Now().UnixNano())
 
@@ -67,7 +63,7 @@ func main() {
 	// If you do not need additional controller, please ignore.
 	controllerInitializers["nodeipam"] = startNodeIpamControllerWrapper
 
-	command := app.NewCloudControllerManagerCommand(ccmOptions, cloudInitializer, controllerInitializers)
+	command := app.NewCloudControllerManagerCommand(ccmOptions, cloudInitializer, controllerInitializers, wait.NeverStop)
 
 	// TODO: once we switch everything over to Cobra commands, we can go back to calling
 	// utilflag.InitFlags() (by removing its pflag.Parse() call). For now, we have to set the
@@ -84,9 +80,9 @@ func main() {
 }
 
 func cloudInitializer(config *cloudcontrollerconfig.CompletedConfig) cloudprovider.Interface {
-	cloudConfigFile := config.ComponentConfig.KubeCloudShared.CloudProvider.CloudConfigFile
+	cloudConfig := config.ComponentConfig.KubeCloudShared.CloudProvider
 	// initialize cloud provider with the cloud provider name and config file provided
-	cloud, err := cloudprovider.InitCloudProvider(cloudProviderName, cloudConfigFile)
+	cloud, err := cloudprovider.InitCloudProvider(cloudConfig.Name, cloudConfig.CloudConfigFile)
 	if err != nil {
 		klog.Fatalf("Cloud provider could not be initialized: %v", err)
 	}

staging/src/k8s.io/cloud-provider/app/controllermanager.go

@@ -64,30 +64,33 @@ const (
 )
 
 // NewCloudControllerManagerCommand creates a *cobra.Command object with default parameters
-func NewCloudControllerManagerCommand(s *options.CloudControllerManagerOptions, cloudInitializer InitCloudFunc, initFuncConstructor map[string]InitFuncConstructor) *cobra.Command {
+func NewCloudControllerManagerCommand(s *options.CloudControllerManagerOptions, cloudInitializer InitCloudFunc, initFuncConstructor map[string]InitFuncConstructor, stopCh <-chan struct{}) *cobra.Command {
 
 	cmd := &cobra.Command{
 		Use: "cloud-controller-manager",
 		Long: `The Cloud controller manager is a daemon that embeds
 the cloud specific control loops shipped with Kubernetes.`,
-		Run: func(cmd *cobra.Command, args []string) {
+		RunE: func(cmd *cobra.Command, args []string) error {
 			verflag.PrintAndExitIfRequested()
 			cliflag.PrintFlags(cmd.Flags())
 
 			c, err := s.Config(ControllerNames(initFuncConstructor), ControllersDisabledByDefault.List())
 			if err != nil {
 				fmt.Fprintf(os.Stderr, "%v\n", err)
-				os.Exit(1)
+				return err
+				//os.Exit(1)
 			}
 
 			completedConfig := c.Complete()
 			cloud := cloudInitializer(completedConfig)
 			controllerInitializers := ConstructControllerInitializers(initFuncConstructor, completedConfig, cloud)
 
-			if err := Run(completedConfig, cloud, controllerInitializers, wait.NeverStop); err != nil {
+			if err := Run(completedConfig, cloud, controllerInitializers, stopCh); err != nil {
 				fmt.Fprintf(os.Stderr, "%v\n", err)
-				os.Exit(1)
+				return err
+				//os.Exit(1)
 			}
+			return nil
 		},
 		Args: func(cmd *cobra.Command, args []string) error {
 			for _, arg := range args {

staging/src/k8s.io/cloud-provider/app/testing/testserver.go

@@ -22,17 +22,18 @@ import (
 	"io/ioutil"
 	"net"
 	"os"
+	"strings"
 	"time"
 
 	"github.com/spf13/pflag"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/kubernetes"
+	restclient "k8s.io/client-go/rest"
 	cloudprovider "k8s.io/cloud-provider"
 	"k8s.io/cloud-provider/app"
 	"k8s.io/cloud-provider/app/config"
 	"k8s.io/cloud-provider/options"
-
+	"k8s.io/component-base/cli/flag"
-	"k8s.io/apimachinery/pkg/util/wait"
-	"k8s.io/client-go/kubernetes"
-	restclient "k8s.io/client-go/rest"
 )
 
 // TearDownFunc is to be called to tear down a test server.
@@ -42,7 +43,7 @@ type TearDownFunc func()
 type TestServer struct {
 	LoopbackClientConfig *restclient.Config // Rest client config using the magic token
 	Options              *options.CloudControllerManagerOptions
-	Config               *config.Config
+	Config               *config.CompletedConfig
 	TearDownFn           TearDownFunc // TearDown function
 	TmpDir               string       // Temp Dir used, by the apiserver
 }
@@ -62,6 +63,8 @@ type Logger interface {
 // 		 enough time to remove temporary files.
 func StartTestServer(t Logger, customFlags []string) (result TestServer, err error) {
 	stopCh := make(chan struct{})
+	configDoneCh := make(chan string)
+	var capturedConfig config.CompletedConfig
 	tearDown := func() {
 		close(stopCh)
 		if len(result.TmpDir) != 0 {
@@ -79,58 +82,94 @@ func StartTestServer(t Logger, customFlags []string) (result TestServer, err err
 		return result, fmt.Errorf("failed to create temp dir: %v", err)
 	}
 
-	fs := pflag.NewFlagSet("test", pflag.PanicOnError)
-
 	s, err := options.NewCloudControllerManagerOptions()
 	if err != nil {
 		return TestServer{}, err
 	}
-	namedFlagSets := s.Flags([]string{}, []string{})
+
-	for _, f := range namedFlagSets.FlagSets {
+	cloudInitializer := func(config *config.CompletedConfig) cloudprovider.Interface {
-		fs.AddFlagSet(f)
+		capturedConfig = *config
-	}
+		configDoneCh <- ""
-	fs.Parse(customFlags)
+		close(configDoneCh)
-	if s.SecureServing.BindPort != 0 {
+		cloudConfig := config.ComponentConfig.KubeCloudShared.CloudProvider
-		s.SecureServing.Listener, s.SecureServing.BindPort, err = createListenerOnFreePort()
+		cloud, err := cloudprovider.InitCloudProvider(cloudConfig.Name, cloudConfig.CloudConfigFile)
 		if err != nil {
-			return result, fmt.Errorf("failed to create listener: %v", err)
+			t.Fatalf("Cloud provider could not be initialized: %v", err)
 		}
 		s.SecureServing.ServerCert.CertDirectory = result.TmpDir
+		if cloud == nil {
+			t.Fatalf("Cloud provider is nil")
+		}
+		return cloud
+	}
+	command := app.NewCloudControllerManagerCommand(s, cloudInitializer, app.DefaultInitFuncConstructors, stopCh)
+	pflag.CommandLine.SetNormalizeFunc(flag.WordSepNormalizeFunc)
 
-		t.Logf("cloud-controller-manager will listen securely on port %d...", s.SecureServing.BindPort)
+	commandArgs := []string{}
+	listeners := []net.Listener{}
+	disableInsecure := false
+	disableSecure := false
+	for _, arg := range customFlags {
+		if strings.HasPrefix(arg, "--secure-port=") {
+			if arg == "--secure-port=0" {
+				commandArgs = append(commandArgs, arg)
+				disableSecure = true
+			}
+		} else if strings.HasPrefix(arg, "--port=") {
+			if arg == "--port=0" {
+				commandArgs = append(commandArgs, arg)
+				disableInsecure = true
+			}
+		} else if strings.HasPrefix(arg, "--cert-dir=") {
+			// skip it
+		} else {
+			commandArgs = append(commandArgs, arg)
+		}
 	}
 
-	if s.InsecureServing.BindPort != 0 {
+	if !disableSecure {
-		s.InsecureServing.Listener, s.InsecureServing.BindPort, err = createListenerOnFreePort()
+		listener, bindPort, err := createListenerOnFreePort()
 		if err != nil {
 			return result, fmt.Errorf("failed to create listener: %v", err)
 		}
+		listeners = append(listeners, listener)
+		commandArgs = append(commandArgs, fmt.Sprintf("--secure-port=%d", bindPort))
+		commandArgs = append(commandArgs, fmt.Sprintf("--cert-dir=%s", result.TmpDir))
 
-		t.Logf("cloud-controller-manager will listen insecurely on port %d...", s.InsecureServing.BindPort)
+		t.Logf("cloud-controller-manager will listen securely on port %d...", bindPort)
 	}
+	if !disableInsecure {
+		listener, bindPort, err := createListenerOnFreePort()
+		if err != nil {
+			return result, fmt.Errorf("failed to create listener: %v", err)
+		}
+		listeners = append(listeners, listener)
+		commandArgs = append(commandArgs, fmt.Sprintf("--port=%d", bindPort))
 
-	config, err := s.Config([]string{}, []string{})
+		t.Logf("cloud-controller-manager will listen securely on port %d...", bindPort)
-	if err != nil {
-		return result, fmt.Errorf("failed to create config from options: %v", err)
 	}
-	cloudConfig := config.Complete().ComponentConfig.KubeCloudShared.CloudProvider
+	for _, listener := range listeners {
-	cloud, err := cloudprovider.InitCloudProvider(cloudConfig.Name, cloudConfig.CloudConfigFile)
+		listener.Close()
-	if err != nil {
-		return result, fmt.Errorf("cloud provider could not be initialized: %v", err)
-	}
-	if cloud == nil {
-		return result, fmt.Errorf("cloud provider is nil")
 	}
 
 	errCh := make(chan error)
-	go func(stopCh <-chan struct{}) {
+	go func() {
-		if err := app.Run(config.Complete(), cloud, app.ConstructControllerInitializers(app.DefaultInitFuncConstructors, config.Complete(), cloud), stopCh); err != nil {
+		command.SetArgs(commandArgs)
+		if err := command.Execute(); err != nil {
 			errCh <- err
 		}
-	}(stopCh)
+		close(errCh)
+	}()
+
+	select {
+	case <-configDoneCh:
+
+	case err := <-errCh:
+		return result, err
+	}
 
 	t.Logf("Waiting for /healthz to be ok...")
-	client, err := kubernetes.NewForConfig(config.LoopbackClientConfig)
+	client, err := kubernetes.NewForConfig(capturedConfig.LoopbackClientConfig)
 	if err != nil {
 		return result, fmt.Errorf("failed to create a client: %v", err)
 	}
@@ -154,9 +193,9 @@ func StartTestServer(t Logger, customFlags []string) (result TestServer, err err
 	}
 
 	// from here the caller must call tearDown
-	result.LoopbackClientConfig = config.LoopbackClientConfig
+	result.LoopbackClientConfig = capturedConfig.LoopbackClientConfig
 	result.Options = s
-	result.Config = config
+	result.Config = &capturedConfig
 	result.TearDownFn = tearDown
 
 	return result, nil

staging/src/k8s.io/cloud-provider/sample/basic_main.go

@@ -21,6 +21,7 @@ limitations under the License.
 package main
 
 import (
+	"k8s.io/apimachinery/pkg/util/wait"
 	"math/rand"
 	"os"
 	"time"
@@ -39,13 +40,6 @@ import (
 	// e.g. _"k8s.io/legacy-cloud-providers/<provider>"
 )
 
-const (
-	// The variables below are samples, please edit the value for your case.
-
-	// sampleCloudProviderName shows an sample of using hard coded parameter for CloudProviderName
-	sampleCloudProviderName = "SampleCloudProviderName"
-)
-
 func main() {
 	rand.Seed(time.Now().UnixNano())
 
@@ -54,7 +48,7 @@ func main() {
 		klog.Fatalf("unable to initialize command options: %v", err)
 	}
 
-	command := app.NewCloudControllerManagerCommand(ccmOptions, cloudInitializer, app.DefaultInitFuncConstructors)
+	command := app.NewCloudControllerManagerCommand(ccmOptions, cloudInitializer, app.DefaultInitFuncConstructors, wait.NeverStop)
 
 	// TODO: once we switch everything over to Cobra commands, we can go back to calling
 	// utilflag.InitFlags() (by removing its pflag.Parse() call). For now, we have to set the
@@ -71,9 +65,10 @@ func main() {
 }
 
 func cloudInitializer(config *config.CompletedConfig) cloudprovider.Interface {
-	cloudConfigFile := config.ComponentConfig.KubeCloudShared.CloudProvider.CloudConfigFile
+	cloudConfig := config.ComponentConfig.KubeCloudShared.CloudProvider
+
 	// initialize cloud provider with the cloud provider name and config file provided
-	cloud, err := cloudprovider.InitCloudProvider(sampleCloudProviderName, cloudConfigFile)
+	cloud, err := cloudprovider.InitCloudProvider(cloudConfig.Name, cloudConfig.CloudConfigFile)
 	if err != nil {
 		klog.Fatalf("Cloud provider could not be initialized: %v", err)
 	}