Added ACI 5.2.7.1 variables

cluster/defaults.go

@@ -119,6 +119,11 @@ const (
 	DefaultAciSleepTimeSnatGlobalInfoSync       = "0"
 	DefaultAciOpflexAgentOpflexAsyncjsonEnabled = "false"
 	DefaultAciOpflexAgentOvsAsyncjsonEnabled    = "false"
+	DefaultAciOpflexAgentPolicyRetryDelayTimer  = "10"
+	DefaultAciAciMultipod                       = "false"
+	DefaultAciAciMultipodUbuntu                 = "false"
+	DefaultAciDhcpRenewMaxRetryCount            = "0"
+	DefaultAciDhcpDelay                         = "0"
 
 	KubeAPIArgAdmissionControlConfigFile             = "admission-control-config-file"
 	DefaultKubeAPIArgAdmissionControlConfigFileValue = "/etc/kubernetes/admission.yaml"
@@ -687,6 +692,11 @@ func (c *Cluster) setClusterNetworkDefaults() {
 			AciSleepTimeSnatGlobalInfoSync:       DefaultAciSleepTimeSnatGlobalInfoSync,
 			AciOpflexAgentOpflexAsyncjsonEnabled: DefaultAciOpflexAgentOpflexAsyncjsonEnabled,
 			AciOpflexAgentOvsAsyncjsonEnabled:    DefaultAciOpflexAgentOvsAsyncjsonEnabled,
+			AciOpflexAgentPolicyRetryDelayTimer:  DefaultAciOpflexAgentPolicyRetryDelayTimer,
+			AciAciMultipod:                       DefaultAciAciMultipod,
+			AciAciMultipodUbuntu:                 DefaultAciAciMultipodUbuntu,
+			AciDhcpRenewMaxRetryCount:            DefaultAciDhcpRenewMaxRetryCount,
+			AciDhcpDelay:                         DefaultAciDhcpDelay,
 		}
 	}
 	if c.Network.CalicoNetworkProvider != nil {
@@ -750,6 +760,11 @@ func (c *Cluster) setClusterNetworkDefaults() {
 		setDefaultIfEmpty(&c.Network.AciNetworkProvider.SleepTimeSnatGlobalInfoSync, DefaultAciSleepTimeSnatGlobalInfoSync)
 		setDefaultIfEmpty(&c.Network.AciNetworkProvider.OpflexAgentOpflexAsyncjsonEnabled, DefaultAciOpflexAgentOpflexAsyncjsonEnabled)
 		setDefaultIfEmpty(&c.Network.AciNetworkProvider.OpflexAgentOvsAsyncjsonEnabled, DefaultAciOpflexAgentOvsAsyncjsonEnabled)
+		setDefaultIfEmpty(&c.Network.AciNetworkProvider.OpflexAgentPolicyRetryDelayTimer, DefaultAciOpflexAgentPolicyRetryDelayTimer)
+		setDefaultIfEmpty(&c.Network.AciNetworkProvider.AciMultipod, DefaultAciAciMultipod)
+		setDefaultIfEmpty(&c.Network.AciNetworkProvider.AciMultipodUbuntu, DefaultAciAciMultipodUbuntu)
+		setDefaultIfEmpty(&c.Network.AciNetworkProvider.DhcpRenewMaxRetryCount, DefaultAciDhcpRenewMaxRetryCount)
+		setDefaultIfEmpty(&c.Network.AciNetworkProvider.DhcpDelay, DefaultAciDhcpDelay)
 		networkPluginConfigDefaultsMap[AciOVSMemoryLimit] = c.Network.AciNetworkProvider.OVSMemoryLimit
 		networkPluginConfigDefaultsMap[AciImagePullPolicy] = c.Network.AciNetworkProvider.ImagePullPolicy
 		networkPluginConfigDefaultsMap[AciPBRTrackingNonSnat] = c.Network.AciNetworkProvider.PBRTrackingNonSnat
@@ -796,6 +811,11 @@ func (c *Cluster) setClusterNetworkDefaults() {
 		networkPluginConfigDefaultsMap[AciSleepTimeSnatGlobalInfoSync] = c.Network.AciNetworkProvider.SleepTimeSnatGlobalInfoSync
 		networkPluginConfigDefaultsMap[AciOpflexAgentOpflexAsyncjsonEnabled] = c.Network.AciNetworkProvider.OpflexAgentOpflexAsyncjsonEnabled
 		networkPluginConfigDefaultsMap[AciOpflexAgentOvsAsyncjsonEnabled] = c.Network.AciNetworkProvider.OpflexAgentOvsAsyncjsonEnabled
+		networkPluginConfigDefaultsMap[AciOpflexAgentPolicyRetryDelayTimer] = c.Network.AciNetworkProvider.OpflexAgentPolicyRetryDelayTimer
+		networkPluginConfigDefaultsMap[AciDhcpRenewMaxRetryCount] = c.Network.AciNetworkProvider.DhcpRenewMaxRetryCount
+		networkPluginConfigDefaultsMap[AciDhcpDelay] = c.Network.AciNetworkProvider.DhcpDelay
+		networkPluginConfigDefaultsMap[AciAciMultipod] = c.Network.AciNetworkProvider.AciMultipod
+		networkPluginConfigDefaultsMap[AciAciMultipodUbuntu] = c.Network.AciNetworkProvider.AciMultipodUbuntu
 		networkPluginConfigDefaultsMap[AciSystemIdentifier] = c.Network.AciNetworkProvider.SystemIdentifier
 		networkPluginConfigDefaultsMap[AciToken] = c.Network.AciNetworkProvider.Token
 		networkPluginConfigDefaultsMap[AciApicUserName] = c.Network.AciNetworkProvider.ApicUserName

cluster/network.go

@@ -154,6 +154,11 @@ const (
 	AciSleepTimeSnatGlobalInfoSync       = "aci_sleep_time_snat_global_info_sync"
 	AciOpflexAgentOpflexAsyncjsonEnabled = "aci_opflex_agent_opflex_asyncjson_enabled"
 	AciOpflexAgentOvsAsyncjsonEnabled    = "aci_opflex_agent_ovs_asyncjson_enabled"
+	AciOpflexAgentPolicyRetryDelayTimer  = "aci_opflex_agent_policy_retry_delay_timer"
+	AciAciMultipod                       = "aci_aci_multipod"
+	AciAciMultipodUbuntu                 = "aci_aci_multipod_ubuntu"
+	AciDhcpRenewMaxRetryCount            = "aci_dhcp_renew_max_retry_count"
+	AciDhcpDelay                         = "aci_dhcp_delay"
 	// List of map keys to be used with network templates
 
 	// EtcdEndpoints is the server address for Etcd, used by calico
@@ -243,15 +248,13 @@ const (
 	AciControllerContainer                 = "AciControllerContainer"
 	AciGbpServerContainer                  = "AciGbpServerContainer"
 	AciOpflexServerContainer               = "AciOpflexServerContainer"
-	StaticServiceIPStart                   = "StaticServiceIPStart"
+	StaticServiceIPPool                    = "StaticServiceIPPool"
-	StaticServiceIPEnd                     = "StaticServiceIPEnd"
+	PodNetwork                             = "PodNetwork"
-	PodGateway                             = "PodGateway"
+	PodSubnet                              = "PodSubnet"
-	PodIPStart                             = "PodIPStart"
+	PodIPPool                              = "PodIPPool"
-	PodIPEnd                               = "PodIPEnd"
 	NodeServiceIPStart                     = "NodeServiceIPStart"
 	NodeServiceIPEnd                       = "NodeServiceIPEnd"
-	ServiceIPStart                         = "ServiceIPStart"
+	ServiceIPPool                          = "ServiceIPPool"
-	ServiceIPEnd                           = "ServiceIPEnd"
 	UseAciCniPriorityClass                 = "UseAciCniPriorityClass"
 	NoPriorityClass                        = "NoPriorityClass"
 	MaxNodesSvcGraph                       = "MaxNodesSvcGraph"
@@ -298,6 +301,11 @@ const (
 	SleepTimeSnatGlobalInfoSync            = "SleepTimeSnatGlobalInfoSync"
 	OpflexAgentOpflexAsyncjsonEnabled      = "OpflexAgentOpflexAsyncjsonEnabled"
 	OpflexAgentOvsAsyncjsonEnabled         = "OpflexAgentOvsAsyncjsonEnabled"
+	OpflexAgentPolicyRetryDelayTimer       = "OpflexAgentPolicyRetryDelayTimer"
+	AciMultipod                            = "AciMultipod"
+	AciMultipodUbuntu                      = "AciMultipodUbuntu"
+	DhcpRenewMaxRetryCount                 = "DhcpRenewMaxRetryCount"
+	DhcpDelay                              = "DhcpDelay"
 	OVSMemoryLimit                         = "OVSMemoryLimit"
 	NodeSubnet                             = "NodeSubnet"
 	NodeSelector                           = "NodeSelector"
@@ -305,6 +313,16 @@ const (
 	Tolerations                            = "Tolerations"
 )
 
+type IPPool struct {
+	Start net.IP
+	End   net.IP
+}
+
+type PodIPNetwork struct {
+	Subnet  net.IPNet
+	Gateway net.IP
+}
+
 var EtcdPortList = []string{
 	EtcdPort1,
 	EtcdPort2,
@@ -323,7 +341,7 @@ var EtcdClientPortList = []string{
 }
 
 var CalicoNetworkLabels = []string{CalicoNodeLabel, CalicoControllerLabel}
-var IPv6CompatibleNetworkPlugins = []string{CalicoNetworkPlugin}
+var IPv6CompatibleNetworkPlugins = []string{CalicoNetworkPlugin, AciNetworkPlugin}
 
 func (c *Cluster) deployNetworkPlugin(ctx context.Context, data map[string]interface{}) error {
 	log.Infof(ctx, "[network] Setting up network plugin: %s", c.Network.Plugin)
@@ -485,26 +503,60 @@ func (c *Cluster) doWeaveDeploy(ctx context.Context, data map[string]interface{}
 }
 
 func (c *Cluster) doAciDeploy(ctx context.Context, data map[string]interface{}) error {
-	_, clusterCIDR, err := net.ParseCIDR(c.ClusterCIDR)
+	var podIPPool []IPPool
-	if err != nil {
+	var podNetwork []PodIPNetwork
-		return err
+	var podSubnet []string
+
+	ClusterCIDRs := strings.Split(c.ClusterCIDR, ",")
+	for _, clusterCIDR := range ClusterCIDRs {
+		podSubnet = append(podSubnet, fmt.Sprintf("\"%s\"", clusterCIDR))
+		_, clusterCIDR, err := net.ParseCIDR(clusterCIDR)
+		if err != nil {
+			return err
+		}
+		podIPStart, podIPEnd := cidr.AddressRange(clusterCIDR)
+		podIPPool = append(podIPPool, IPPool{Start: cidr.Inc(cidr.Inc(podIPStart)), End: cidr.Dec(podIPEnd)})
+		podNetwork = append(podNetwork, PodIPNetwork{Subnet: *clusterCIDR, Gateway: cidr.Inc(podIPStart)})
 	}
-	podIPStart, podIPEnd := cidr.AddressRange(clusterCIDR)
+
-	_, staticExternalSubnet, err := net.ParseCIDR(c.Network.Options[AciStaticExternalSubnet])
+	var staticServiceIPPool []IPPool
-	if err != nil {
+	var staticExtern []string
-		return err
+	staticExternalSubnets := strings.Split(c.Network.Options[AciStaticExternalSubnet], ",")
+	for _, staticExternalSubnet := range staticExternalSubnets {
+		staticExtern = append(staticExtern, fmt.Sprintf("\"%s\"", staticExternalSubnet))
+		_, externStatic, err := net.ParseCIDR(staticExternalSubnet)
+		if err != nil {
+			return err
+		}
+		staticServiceIPStart, staticServiceIPEnd := cidr.AddressRange(externStatic)
+		staticServiceIPPool = append(staticServiceIPPool, IPPool{Start: cidr.Inc(cidr.Inc(staticServiceIPStart)), End: cidr.Dec(staticServiceIPEnd)})
 	}
-	staticServiceIPStart, staticServiceIPEnd := cidr.AddressRange(staticExternalSubnet)
+
 	_, svcGraphSubnet, err := net.ParseCIDR(c.Network.Options[AciServiceGraphSubnet])
 	if err != nil {
 		return err
 	}
 	nodeServiceIPStart, nodeServiceIPEnd := cidr.AddressRange(svcGraphSubnet)
-	_, dynamicExternalSubnet, err := net.ParseCIDR(c.Network.Options[AciDynamicExternalSubnet])
+
-	if err != nil {
+	var serviceIPPool []IPPool
-		return err
+	var dynamicExtern []string
+	dynamicExternalSubnets := strings.Split(c.Network.Options[AciDynamicExternalSubnet], ",")
+	for _, dynamicExternalSubnet := range dynamicExternalSubnets {
+		dynamicExtern = append(dynamicExtern, fmt.Sprintf("\"%s\"", dynamicExternalSubnet))
+		_, externDynamic, err := net.ParseCIDR(dynamicExternalSubnet)
+		if err != nil {
+			return err
+		}
+		serviceIPStart, serviceIPEnd := cidr.AddressRange(externDynamic)
+		serviceIPPool = append(serviceIPPool, IPPool{Start: cidr.Inc(cidr.Inc(serviceIPStart)), End: cidr.Dec(serviceIPEnd)})
 	}
-	serviceIPStart, serviceIPEnd := cidr.AddressRange(dynamicExternalSubnet)
+
+	var nodeSubnets []string
+	NodeSubnets := strings.Split(c.Network.Options[AciNodeSubnet], ",")
+	for _, nodeSubnet := range NodeSubnets {
+		nodeSubnets = append(nodeSubnets, fmt.Sprintf("\"%s\"", nodeSubnet))
+	}
+
 	if c.Network.Options[AciTenant] == "" {
 		c.Network.Options[AciTenant] = c.Network.Options[AciSystemIdentifier]
 	}
@@ -522,14 +574,14 @@ func (c *Cluster) doAciDeploy(ctx context.Context, data map[string]interface{})
 		EncapType:                         c.Network.Options[AciEncapType],
 		McastRangeStart:                   c.Network.Options[AciMcastRangeStart],
 		McastRangeEnd:                     c.Network.Options[AciMcastRangeEnd],
-		NodeSubnet:                        c.Network.Options[AciNodeSubnet],
+		NodeSubnet:                        nodeSubnets,
 		AEP:                               c.Network.Options[AciAEP],
 		VRFName:                           c.Network.Options[AciVRFName],
 		VRFTenant:                         c.Network.Options[AciVRFTenant],
 		L3Out:                             c.Network.Options[AciL3Out],
 		L3OutExternalNetworks:             c.Network.AciNetworkProvider.L3OutExternalNetworks,
-		DynamicExternalSubnet:             c.Network.Options[AciDynamicExternalSubnet],
+		DynamicExternalSubnet:             dynamicExtern,
-		StaticExternalSubnet:              c.Network.Options[AciStaticExternalSubnet],
+		StaticExternalSubnet:              staticExtern,
 		ServiceGraphSubnet:                c.Network.Options[AciServiceGraphSubnet],
 		KubeAPIVlan:                       c.Network.Options[AciKubeAPIVlan],
 		ServiceVlan:                       c.Network.Options[AciServiceVlan],
@@ -547,15 +599,13 @@ func (c *Cluster) doAciDeploy(ctx context.Context, data map[string]interface{})
 		OpflexAgentLogLevel:               c.Network.Options[AciOpflexAgentLogLevel],
 		OVSMemoryLimit:                    c.Network.Options[AciOVSMemoryLimit],
 		ClusterCIDR:                       c.ClusterCIDR,
-		StaticServiceIPStart:              cidr.Inc(cidr.Inc(staticServiceIPStart)),
+		PodNetwork:                        podNetwork,
-		StaticServiceIPEnd:                cidr.Dec(staticServiceIPEnd),
+		PodIPPool:                         podIPPool,
-		PodGateway:                        cidr.Inc(podIPStart),
+		StaticServiceIPPool:               staticServiceIPPool,
-		PodIPStart:                        cidr.Inc(cidr.Inc(podIPStart)),
+		ServiceIPPool:                     serviceIPPool,
-		PodIPEnd:                          cidr.Dec(podIPEnd),
+		PodSubnet:                         podSubnet,
 		NodeServiceIPStart:                cidr.Inc(cidr.Inc(nodeServiceIPStart)),
 		NodeServiceIPEnd:                  cidr.Dec(nodeServiceIPEnd),
-		ServiceIPStart:                    cidr.Inc(cidr.Inc(serviceIPStart)),
-		ServiceIPEnd:                      cidr.Dec(serviceIPEnd),
 		UseAciCniPriorityClass:            c.Network.Options[AciUseAciCniPriorityClass],
 		NoPriorityClass:                   c.Network.Options[AciNoPriorityClass],
 		MaxNodesSvcGraph:                  c.Network.Options[AciMaxNodesSvcGraph],
@@ -602,6 +652,11 @@ func (c *Cluster) doAciDeploy(ctx context.Context, data map[string]interface{})
 		SleepTimeSnatGlobalInfoSync:       c.Network.Options[AciSleepTimeSnatGlobalInfoSync],
 		OpflexAgentOpflexAsyncjsonEnabled: c.Network.Options[AciOpflexAgentOpflexAsyncjsonEnabled],
 		OpflexAgentOvsAsyncjsonEnabled:    c.Network.Options[AciOpflexAgentOvsAsyncjsonEnabled],
+		OpflexAgentPolicyRetryDelayTimer:  c.Network.Options[AciOpflexAgentPolicyRetryDelayTimer],
+		AciMultipod:                       c.Network.Options[AciAciMultipod],
+		AciMultipodUbuntu:                 c.Network.Options[AciAciMultipodUbuntu],
+		DhcpRenewMaxRetryCount:            c.Network.Options[AciDhcpRenewMaxRetryCount],
+		DhcpDelay:                         c.Network.Options[AciDhcpDelay],
 		AciCniDeployContainer:             c.SystemImages.AciCniDeployContainer,
 		AciHostContainer:                  c.SystemImages.AciHostContainer,
 		AciOpflexContainer:                c.SystemImages.AciOpflexContainer,

cluster/validation.go

@@ -231,6 +231,23 @@ func validateNetworkOptions(c *Cluster) error {
 		if !IPv6CompatibleNetworkPluginFound {
 			return fmt.Errorf("Network plugin [%s] does not support IPv6 (dualstack)", c.Network.Plugin)
 		}
+		if c.Network.Plugin == AciNetworkPlugin {
+			k8sVersion := c.RancherKubernetesEngineConfig.Version
+			toMatch, err := semver.Make(k8sVersion[1:])
+			if err != nil {
+				return fmt.Errorf("Cluster version [%s] is not valid semver", k8sVersion)
+			}
+			logrus.Debugf("Checking if cluster version [%s] has dualstack supported aci cni version", k8sVersion)
+			//k8s version needs to have aci version >= 5.2.7.1
+			clusterDualstackAciRange, err := semver.ParseRange(">=1.23.16-rancher2-3 <=1.23.99 || >=1.24.13-rancher2-2")
+			if err != nil {
+				return errors.New("Failed to parse semver range for checking dualstack supported aci cni versions")
+			}
+			if !clusterDualstackAciRange(toMatch) {
+				return fmt.Errorf("Cluster version [%s] does not have dualstack supported aci cni version", k8sVersion)
+			}
+			logrus.Debugf("Cluster version [%s] has dualstack supported aci cni version", k8sVersion)
+		}
 	}
 
 	if c.Network.Plugin == AciNetworkPlugin {
@@ -249,8 +266,7 @@ func validateNetworkOptions(c *Cluster) error {
 		networkOptionsList := []string{AciSystemIdentifier, AciToken, AciApicUserName, AciApicUserKey,
 			AciApicUserCrt, AciEncapType, AciMcastRangeStart, AciMcastRangeEnd,
 			AciNodeSubnet, AciAEP, AciVRFName, AciVRFTenant, AciL3Out, AciDynamicExternalSubnet,
-			AciStaticExternalSubnet, AciServiceGraphSubnet, AciKubeAPIVlan, AciServiceVlan, AciInfraVlan,
+			AciStaticExternalSubnet, AciServiceGraphSubnet, AciKubeAPIVlan, AciServiceVlan, AciInfraVlan}
-			AciNodeSubnet}
 		for _, v := range networkOptionsList {
 			val, ok := c.Network.Options[v]
 			if !ok || val == "" {

types/rke_types.go

@@ -671,6 +671,11 @@ type AciNetworkProvider struct {
 	SleepTimeSnatGlobalInfoSync       string              `yaml:"sleep_time_snat_global_info_sync,omitempty" json:"sleepTimeSnatGlobalInfoSync,omitempty"`
 	OpflexAgentOpflexAsyncjsonEnabled string              `yaml:"opflex_agent_opflex_asyncjson_enabled,omitempty" json:"opflexAgentOpflexAsyncjsonEnabled,omitempty"`
 	OpflexAgentOvsAsyncjsonEnabled    string              `yaml:"opflex_agent_ovs_asyncjson_enabled,omitempty" json:"opflexAgentOvsAsyncjsonEnabled,omitempty"`
+	OpflexAgentPolicyRetryDelayTimer  string              `yaml:"opflex_agent_policy_retry_delay_timer,omitempty" json:"opflexAgentPolicyRetryDelayTimer,omitempty"`
+	AciMultipod                       string              `yaml:"aci_multipod,omitempty" json:"aciMultipod,omitempty"`
+	AciMultipodUbuntu                 string              `yaml:"aci_multipod_ubuntu,omitempty" json:"aciMultipodUbuntu,omitempty"`
+	DhcpRenewMaxRetryCount            string              `yaml:"dhcp_renew_max_retry_count,omitempty" json:"dhcpRenewMaxRetryCount,omitempty"`
+	DhcpDelay                         string              `yaml:"dhcp_delay,omitempty" json:"dhcpDelay,omitempty"`
 }
 
 type KubernetesServicesOptions struct {