Adding config extension to CCM.

2025-09-10 05:30:26 +00:00 · 2020-08-03 03:27:45 -07:00
parent 4b65f70652
commit 895a0a8d5e
14 changed files with 732 additions and 60 deletions
--- a/cmd/cloud-controller-manager/.import-restrictions
+++ b/cmd/cloud-controller-manager/.import-restrictions
@@ -1,4 +1,37 @@
 rules:
  - selectorRegexp: k8s[.]io/kubernetes
    allowedPrefixes:
-      - k8s.io/kubernetes/cmd/cloud-controller-manager
+      - k8s.io/kubernetes/pkg/api/legacyscheme
+      - k8s.io/kubernetes/pkg/api/service
+      - k8s.io/kubernetes/pkg/api/v1/pod
+      - k8s.io/kubernetes/pkg/apis/apps
+      - k8s.io/kubernetes/pkg/apis/autoscaling
+      - k8s.io/kubernetes/pkg/apis/core
+      - k8s.io/kubernetes/pkg/apis/core/helper
+      - k8s.io/kubernetes/pkg/apis/core/install
+      - k8s.io/kubernetes/pkg/apis/core/pods
+      - k8s.io/kubernetes/pkg/apis/core/v1
+      - k8s.io/kubernetes/pkg/apis/core/v1/helper
+      - k8s.io/kubernetes/pkg/apis/core/validation
+      - k8s.io/kubernetes/pkg/apis/scheduling
+      - k8s.io/kubernetes/pkg/capabilities
+      - k8s.io/kubernetes/pkg/cluster/ports
+      - k8s.io/kubernetes/pkg/controller
+      - k8s.io/kubernetes/pkg/controller/nodeipam
+      - k8s.io/kubernetes/pkg/controller/nodeipam/config
+      - k8s.io/kubernetes/pkg/controller/nodeipam/ipam
+      - k8s.io/kubernetes/pkg/controller/nodeipam/ipam/cidrset
+      - k8s.io/kubernetes/pkg/controller/nodeipam/ipam/sync
+      - k8s.io/kubernetes/pkg/controller/nodeipam/ipam/test
+      - k8s.io/kubernetes/pkg/controller/testutil
+      - k8s.io/kubernetes/pkg/controller/util/node
+      - k8s.io/kubernetes/pkg/features
+      - k8s.io/kubernetes/pkg/fieldpath
+      - k8s.io/kubernetes/pkg/kubelet/types
+      - k8s.io/kubernetes/pkg/kubelet/util/format
+      - k8s.io/kubernetes/pkg/security/apparmor
+      - k8s.io/kubernetes/pkg/securitycontext
+      - k8s.io/kubernetes/pkg/util/hash
+      - k8s.io/kubernetes/pkg/util/node
+      - k8s.io/kubernetes/pkg/util/parsers
+      - k8s.io/kubernetes/pkg/util/taints
--- a/cmd/cloud-controller-manager/BUILD
+++ b/cmd/cloud-controller-manager/BUILD
@@ -18,19 +18,32 @@ go_library(
    name = "go_default_library",
    srcs = [
        "controller-manager.go",
+        "nodeipamcontroller.go",
        "providers.go",
    ],
    importpath = "k8s.io/kubernetes/cmd/cloud-controller-manager",
    deps = [
+        "//pkg/controller/nodeipam:go_default_library",
+        "//pkg/controller/nodeipam/config:go_default_library",
+        "//pkg/controller/nodeipam/ipam:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
+        "//staging/src/k8s.io/cloud-provider:go_default_library",
        "//staging/src/k8s.io/cloud-provider/app:go_default_library",
+        "//staging/src/k8s.io/cloud-provider/app/config:go_default_library",
+        "//staging/src/k8s.io/cloud-provider/options:go_default_library",
+        "//staging/src/k8s.io/component-base/cli/flag:go_default_library",
        "//staging/src/k8s.io/component-base/logs:go_default_library",
        "//staging/src/k8s.io/component-base/metrics/prometheus/clientgo:go_default_library",
        "//staging/src/k8s.io/component-base/metrics/prometheus/version:go_default_library",
+        "//staging/src/k8s.io/controller-manager/app:go_default_library",
        "//staging/src/k8s.io/legacy-cloud-providers/aws:go_default_library",
        "//staging/src/k8s.io/legacy-cloud-providers/azure:go_default_library",
        "//staging/src/k8s.io/legacy-cloud-providers/gce:go_default_library",
        "//staging/src/k8s.io/legacy-cloud-providers/openstack:go_default_library",
        "//staging/src/k8s.io/legacy-cloud-providers/vsphere:go_default_library",
+        "//vendor/github.com/spf13/pflag:go_default_library",
+        "//vendor/k8s.io/klog/v2:go_default_library",
+        "//vendor/k8s.io/utils/net:go_default_library",
    ],
 )

--- a/cmd/cloud-controller-manager/controller-manager.go
+++ b/cmd/cloud-controller-manager/controller-manager.go
@@ -17,32 +17,130 @@ limitations under the License.
 // The external controller manager is responsible for running controller loops that
 // are cloud provider dependent. It uses the API to listen to new events on resources.

+// This file should be written by each cloud provider.
+// The current file demonstrate how other cloud provider should leverage CCM and it uses fake parameters. Please modify for your own use.
+
 package main

 import (
+	"fmt"
 	"math/rand"
+	"net/http"
 	"os"
 	"time"

+	"github.com/spf13/pflag"
+	cloudprovider "k8s.io/cloud-provider"
 	"k8s.io/cloud-provider/app"
+	cloudcontrollerconfig "k8s.io/cloud-provider/app/config"
+	"k8s.io/cloud-provider/options"
+	"k8s.io/component-base/cli/flag"
 	"k8s.io/component-base/logs"
 	_ "k8s.io/component-base/metrics/prometheus/clientgo" // load all the prometheus client-go plugins
 	_ "k8s.io/component-base/metrics/prometheus/version"  // for version metric registration
+	genericcontrollermanager "k8s.io/controller-manager/app"
+	"k8s.io/klog/v2"
+	nodeipamconfig "k8s.io/kubernetes/pkg/controller/nodeipam/config"
+)
+
+const (
+	// cloudProviderName shows an sample of using hard coded parameter
+	cloudProviderName = "SampleCloudProviderName"
+
+	// defaultNodeMaskCIDRIPv4 is default mask size for IPv4 node cidr
+	defaultNodeMaskCIDRIPv4 = 24
+	// defaultNodeMaskCIDRIPv6 is default mask size for IPv6 node cidr
+	defaultNodeMaskCIDRIPv6 = 64
 )

 func main() {
 	rand.Seed(time.Now().UnixNano())

-	command := app.NewCloudControllerManagerCommand()
+	// cloudProviderConfigFile shows an sample of parse config file from flag option
+	var flagset *pflag.FlagSet = pflag.NewFlagSet("flagSet", pflag.ContinueOnError)
+	var cloudProviderConfigFile *string = flagset.String("cloud-provider-configfile", "", "This is the sample input for cloud provider config file")
+	pflag.CommandLine.ParseErrorsWhitelist.UnknownFlags = true
+	_ = pflag.CommandLine.Parse(os.Args[1:])
+
+	// this is an example of allow-listing specific controller loops
+	controllerList := []string{"cloud-node", "cloud-node-lifecycle", "service", "route"}
+
+	s, err := options.NewCloudControllerManagerOptions()
+	if err != nil {
+		klog.Fatalf("unable to initialize command options: %v", err)
+	}
+	c, err := s.Config(controllerList, app.ControllersDisabledByDefault.List())
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "%v\n", err)
+		os.Exit(1)
+	}
+
+	cloud, err := cloudprovider.InitCloudProvider(cloudProviderName, *cloudProviderConfigFile)
+	if err != nil {
+		klog.Fatalf("Cloud provider could not be initialized: %v", err)
+	}
+	if cloud == nil {
+		klog.Fatalf("cloud provider is nil")
+	}
+
+	if !cloud.HasClusterID() {
+		if c.ComponentConfig.KubeCloudShared.AllowUntaggedCloud {
+			klog.Warning("detected a cluster without a ClusterID.  A ClusterID will be required in the future.  Please tag your cluster to avoid any future issues")
+		} else {
+			klog.Fatalf("no ClusterID found.  A ClusterID is required for the cloud provider to function properly.  This check can be bypassed by setting the allow-untagged-cloud option")
+		}
+	}
+
+	// Initialize the cloud provider with a reference to the clientBuilder
+	cloud.Initialize(c.ClientBuilder, make(chan struct{}))
+	// Set the informer on the user cloud object
+	if informerUserCloud, ok := cloud.(cloudprovider.InformerUser); ok {
+		informerUserCloud.SetInformers(c.SharedInformers)
+	}
+
+	controllerInitializers := app.DefaultControllerInitializers(c.Complete(), cloud)
+
+	// Here is an example to remove the controller which is not needed.
+	// e.g. remove the cloud-node-lifecycle controller which current cloud provider does not need.
+	//delete(controllerInitializers, "cloud-node-lifecycle")
+
+	// Here is an example to add an controller(NodeIpamController) which will be used by cloud provider
+	// generate nodeipamconfig. Here is an sample code. Please pass the right parameter in your code.
+	// If you do not need additional controller, please ignore.
+	nodeipamconfig := nodeipamconfig.NodeIPAMControllerConfiguration{
+		ServiceCIDR:          "sample",
+		SecondaryServiceCIDR: "sample",
+		NodeCIDRMaskSize:     11,
+		NodeCIDRMaskSizeIPv4: 11,
+		NodeCIDRMaskSizeIPv6: 111,
+	}
+	controllerInitializers["nodeipam"] = startNodeIpamControllerWrapper(c.Complete(), nodeipamconfig, cloud)
+
+	command := app.NewCloudControllerManagerCommand(s, c, controllerInitializers)

 	// TODO: once we switch everything over to Cobra commands, we can go back to calling
 	// utilflag.InitFlags() (by removing its pflag.Parse() call). For now, we have to set the
 	// normalize func and add the go flag set by hand.
+	// Here is an sample
+	pflag.CommandLine.SetNormalizeFunc(flag.WordSepNormalizeFunc)
 	// utilflag.InitFlags()
 	logs.InitLogs()
 	defer logs.FlushLogs()

+	// the flags could be set before execute
+	command.Flags().VisitAll(func(flag *pflag.Flag) {
+		if flag.Name == "cloud-provider" {
+			flag.Value.Set("SampleCloudProviderFlagValue")
+			return
+		}
+	})
 	if err := command.Execute(); err != nil {
 		os.Exit(1)
 	}
 }
+
+func startNodeIpamControllerWrapper(ccmconfig *cloudcontrollerconfig.CompletedConfig, nodeipamconfig nodeipamconfig.NodeIPAMControllerConfiguration, cloud cloudprovider.Interface) func(ctx genericcontrollermanager.ControllerContext) (http.Handler, bool, error) {
+	return func(ctx genericcontrollermanager.ControllerContext) (http.Handler, bool, error) {
+		return startNodeIpamController(ccmconfig, nodeipamconfig, ctx, cloud)
+	}
+}
--- a/cmd/cloud-controller-manager/nodeipamcontroller.go
+++ b/cmd/cloud-controller-manager/nodeipamcontroller.go
@@ -0,0 +1,203 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// This file holds the code related with the sample nodeipamcontroller
+// which demonstrates how cloud providers add external controllers to cloud-controller-manager
+
+package main
+
+import (
+	"errors"
+	"fmt"
+	"net"
+	"net/http"
+	"strings"
+
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	cloudprovider "k8s.io/cloud-provider"
+	"k8s.io/cloud-provider/app"
+	cloudcontrollerconfig "k8s.io/cloud-provider/app/config"
+	genericcontrollermanager "k8s.io/controller-manager/app"
+	"k8s.io/klog/v2"
+	nodeipamcontroller "k8s.io/kubernetes/pkg/controller/nodeipam"
+	nodeipamconfig "k8s.io/kubernetes/pkg/controller/nodeipam/config"
+	"k8s.io/kubernetes/pkg/controller/nodeipam/ipam"
+	netutils "k8s.io/utils/net"
+)
+
+func startNodeIpamController(ccmconfig *cloudcontrollerconfig.CompletedConfig, nodeipamconfig nodeipamconfig.NodeIPAMControllerConfiguration, ctx genericcontrollermanager.ControllerContext, cloud cloudprovider.Interface) (http.Handler, bool, error) {
+	var serviceCIDR *net.IPNet
+	var secondaryServiceCIDR *net.IPNet
+
+	// should we start nodeIPAM
+	if !ccmconfig.ComponentConfig.KubeCloudShared.AllocateNodeCIDRs {
+		return nil, false, nil
+	}
+
+	// failure: bad cidrs in config
+	clusterCIDRs, dualStack, err := processCIDRs(ccmconfig.ComponentConfig.KubeCloudShared.ClusterCIDR)
+	if err != nil {
+		return nil, false, err
+	}
+
+	// failure: more than one cidr and dual stack is not enabled
+	if len(clusterCIDRs) > 1 && !utilfeature.DefaultFeatureGate.Enabled(app.IPv6DualStack) {
+		return nil, false, fmt.Errorf("len of ClusterCIDRs==%v and dualstack feature is not enabled", len(clusterCIDRs))
+	}
+
+	// failure: more than one cidr but they are not configured as dual stack
+	if len(clusterCIDRs) > 1 && !dualStack {
+		return nil, false, fmt.Errorf("len of ClusterCIDRs==%v and they are not configured as dual stack (at least one from each IPFamily", len(clusterCIDRs))
+	}
+
+	// failure: more than cidrs is not allowed even with dual stack
+	if len(clusterCIDRs) > 2 {
+		return nil, false, fmt.Errorf("len of clusters is:%v > more than max allowed of 2", len(clusterCIDRs))
+	}
+
+	// service cidr processing
+	if len(strings.TrimSpace(nodeipamconfig.ServiceCIDR)) != 0 {
+		_, serviceCIDR, err = net.ParseCIDR(nodeipamconfig.ServiceCIDR)
+		if err != nil {
+			klog.Warningf("Unsuccessful parsing of service CIDR %v: %v", nodeipamconfig.ServiceCIDR, err)
+		}
+	}
+
+	if len(strings.TrimSpace(nodeipamconfig.SecondaryServiceCIDR)) != 0 {
+		_, secondaryServiceCIDR, err = net.ParseCIDR(nodeipamconfig.SecondaryServiceCIDR)
+		if err != nil {
+			klog.Warningf("Unsuccessful parsing of service CIDR %v: %v", nodeipamconfig.SecondaryServiceCIDR, err)
+		}
+	}
+
+	// the following checks are triggered if both serviceCIDR and secondaryServiceCIDR are provided
+	if serviceCIDR != nil && secondaryServiceCIDR != nil {
+		// should have dual stack flag enabled
+		if !utilfeature.DefaultFeatureGate.Enabled(app.IPv6DualStack) {
+			return nil, false, fmt.Errorf("secondary service cidr is provided and IPv6DualStack feature is not enabled")
+		}
+
+		// should be dual stack (from different IPFamilies)
+		dualstackServiceCIDR, err := netutils.IsDualStackCIDRs([]*net.IPNet{serviceCIDR, secondaryServiceCIDR})
+		if err != nil {
+			return nil, false, fmt.Errorf("failed to perform dualstack check on serviceCIDR and secondaryServiceCIDR error:%v", err)
+		}
+		if !dualstackServiceCIDR {
+			return nil, false, fmt.Errorf("serviceCIDR and secondaryServiceCIDR are not dualstack (from different IPfamiles)")
+		}
+	}
+
+	var nodeCIDRMaskSizeIPv4, nodeCIDRMaskSizeIPv6 int
+	if utilfeature.DefaultFeatureGate.Enabled(app.IPv6DualStack) {
+		// only --node-cidr-mask-size-ipv4 and --node-cidr-mask-size-ipv6 supported with dual stack clusters.
+		// --node-cidr-mask-size flag is incompatible with dual stack clusters.
+		nodeCIDRMaskSizeIPv4, nodeCIDRMaskSizeIPv6, err = setNodeCIDRMaskSizesDualStack(nodeipamconfig)
+	} else {
+		// only --node-cidr-mask-size supported with single stack clusters.
+		// --node-cidr-mask-size-ipv4 and --node-cidr-mask-size-ipv6 flags are incompatible with dual stack clusters.
+		nodeCIDRMaskSizeIPv4, nodeCIDRMaskSizeIPv6, err = setNodeCIDRMaskSizes(nodeipamconfig)
+	}
+
+	if err != nil {
+		return nil, false, err
+	}
+
+	// get list of node cidr mask sizes
+	nodeCIDRMaskSizes := getNodeCIDRMaskSizes(clusterCIDRs, nodeCIDRMaskSizeIPv4, nodeCIDRMaskSizeIPv6)
+
+	nodeIpamController, err := nodeipamcontroller.NewNodeIpamController(
+		ctx.InformerFactory.Core().V1().Nodes(),
+		cloud,
+		ctx.ClientBuilder.ClientOrDie("node-controller"),
+		clusterCIDRs,
+		serviceCIDR,
+		secondaryServiceCIDR,
+		nodeCIDRMaskSizes,
+		ipam.CIDRAllocatorType(ccmconfig.ComponentConfig.KubeCloudShared.CIDRAllocatorType),
+	)
+	if err != nil {
+		return nil, true, err
+	}
+	go nodeIpamController.Run(ctx.Stop)
+	return nil, true, nil
+}
+
+// processCIDRs is a helper function that works on a comma separated cidrs and returns
+// a list of typed cidrs
+// a flag if cidrs represents a dual stack
+// error if failed to parse any of the cidrs
+func processCIDRs(cidrsList string) ([]*net.IPNet, bool, error) {
+	cidrsSplit := strings.Split(strings.TrimSpace(cidrsList), ",")
+
+	cidrs, err := netutils.ParseCIDRs(cidrsSplit)
+	if err != nil {
+		return nil, false, err
+	}
+
+	// if cidrs has an error then the previous call will fail
+	// safe to ignore error checking on next call
+	dualstack, _ := netutils.IsDualStackCIDRs(cidrs)
+
+	return cidrs, dualstack, nil
+}
+
+// setNodeCIDRMaskSizes returns the IPv4 and IPv6 node cidr mask sizes.
+// If --node-cidr-mask-size not set, then it will return default IPv4 and IPv6 cidr mask sizes.
+func setNodeCIDRMaskSizes(cfg nodeipamconfig.NodeIPAMControllerConfiguration) (int, int, error) {
+	ipv4Mask, ipv6Mask := defaultNodeMaskCIDRIPv4, defaultNodeMaskCIDRIPv6
+	// NodeCIDRMaskSizeIPv4 and NodeCIDRMaskSizeIPv6 can be used only for dual-stack clusters
+	if cfg.NodeCIDRMaskSizeIPv4 != 0 || cfg.NodeCIDRMaskSizeIPv6 != 0 {
+		return ipv4Mask, ipv6Mask, errors.New("usage of --node-cidr-mask-size-ipv4 and --node-cidr-mask-size-ipv6 are not allowed with non dual-stack clusters")
+	}
+	if cfg.NodeCIDRMaskSize != 0 {
+		ipv4Mask = int(cfg.NodeCIDRMaskSize)
+		ipv6Mask = int(cfg.NodeCIDRMaskSize)
+	}
+	return ipv4Mask, ipv6Mask, nil
+}
+
+// setNodeCIDRMaskSizesDualStack returns the IPv4 and IPv6 node cidr mask sizes to the value provided
+// for --node-cidr-mask-size-ipv4 and --node-cidr-mask-size-ipv6 respectively. If value not provided,
+// then it will return default IPv4 and IPv6 cidr mask sizes.
+func setNodeCIDRMaskSizesDualStack(cfg nodeipamconfig.NodeIPAMControllerConfiguration) (int, int, error) {
+	ipv4Mask, ipv6Mask := defaultNodeMaskCIDRIPv4, defaultNodeMaskCIDRIPv6
+	// NodeCIDRMaskSize can be used only for single stack clusters
+	if cfg.NodeCIDRMaskSize != 0 {
+		return ipv4Mask, ipv6Mask, errors.New("usage of --node-cidr-mask-size is not allowed with dual-stack clusters")
+	}
+	if cfg.NodeCIDRMaskSizeIPv4 != 0 {
+		ipv4Mask = int(cfg.NodeCIDRMaskSizeIPv4)
+	}
+	if cfg.NodeCIDRMaskSizeIPv6 != 0 {
+		ipv6Mask = int(cfg.NodeCIDRMaskSizeIPv6)
+	}
+	return ipv4Mask, ipv6Mask, nil
+}
+
+// getNodeCIDRMaskSizes is a helper function that helps the generate the node cidr mask
+// sizes slice based on the cluster cidr slice
+func getNodeCIDRMaskSizes(clusterCIDRs []*net.IPNet, maskSizeIPv4, maskSizeIPv6 int) []int {
+	nodeMaskCIDRs := make([]int, len(clusterCIDRs))
+
+	for idx, clusterCIDR := range clusterCIDRs {
+		if netutils.IsIPv6CIDR(clusterCIDR) {
+			nodeMaskCIDRs[idx] = maskSizeIPv6
+		} else {
+			nodeMaskCIDRs[idx] = maskSizeIPv4
+		}
+	}
+	return nodeMaskCIDRs
+}