mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-24 20:24:09 +00:00
Merge pull request #50836 from FengyunPan/auto-get-NodeSecurityGroupID
Automatic merge from submit-queue. If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>. Support autoprobing node-security-group for openstack cloud provider 1. Support autoprobing node-security-group 2. Support multiple Security Groups for cluster's nodes 3. Fix recreating Security Group for cluster's nodes This is a part of #50726 **Special notes for your reviewer**: /assign @anguslees /assign @dims **Release note**: ```release-note Support autoprobing node-security-group for openstack cloud provider, Support multiple Security Groups for cluster's nodes. ```
This commit is contained in:
commit
f60b92a26a
@ -56,6 +56,7 @@ go_library(
|
|||||||
"//vendor/k8s.io/api/core/v1:go_default_library",
|
"//vendor/k8s.io/api/core/v1:go_default_library",
|
||||||
"//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
|
"//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
|
||||||
"//vendor/k8s.io/apimachinery/pkg/util/net:go_default_library",
|
"//vendor/k8s.io/apimachinery/pkg/util/net:go_default_library",
|
||||||
|
"//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
|
||||||
"//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
|
"//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
|
||||||
"//vendor/k8s.io/client-go/util/cert:go_default_library",
|
"//vendor/k8s.io/client-go/util/cert:go_default_library",
|
||||||
"//vendor/k8s.io/utils/exec:go_default_library",
|
"//vendor/k8s.io/utils/exec:go_default_library",
|
||||||
|
@ -87,7 +87,7 @@ type LoadBalancerOpts struct {
|
|||||||
MonitorTimeout MyDuration `gcfg:"monitor-timeout"`
|
MonitorTimeout MyDuration `gcfg:"monitor-timeout"`
|
||||||
MonitorMaxRetries uint `gcfg:"monitor-max-retries"`
|
MonitorMaxRetries uint `gcfg:"monitor-max-retries"`
|
||||||
ManageSecurityGroups bool `gcfg:"manage-security-groups"`
|
ManageSecurityGroups bool `gcfg:"manage-security-groups"`
|
||||||
NodeSecurityGroupID string `gcfg:"node-security-group"`
|
NodeSecurityGroupIDs []string // Do not specify, get it automatically when enable manage-security-groups. TODO(FengyunPan): move it into cache
|
||||||
}
|
}
|
||||||
|
|
||||||
type BlockStorageOpts struct {
|
type BlockStorageOpts struct {
|
||||||
@ -248,13 +248,6 @@ func checkOpenStackOpts(openstackOpts *OpenStack) error {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// if enable ManageSecurityGroups, node-security-group should be set.
|
|
||||||
if lbOpts.ManageSecurityGroups {
|
|
||||||
if len(lbOpts.NodeSecurityGroupID) == 0 {
|
|
||||||
return fmt.Errorf("node-security-group not set in cloud provider config")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := checkMetadataSearchOrder(openstackOpts.metadataOpts.SearchOrder); err != nil {
|
if err := checkMetadataSearchOrder(openstackOpts.metadataOpts.SearchOrder); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
@ -40,6 +40,8 @@ import (
|
|||||||
"github.com/gophercloud/gophercloud/pagination"
|
"github.com/gophercloud/gophercloud/pagination"
|
||||||
|
|
||||||
"k8s.io/api/core/v1"
|
"k8s.io/api/core/v1"
|
||||||
|
"k8s.io/apimachinery/pkg/types"
|
||||||
|
"k8s.io/apimachinery/pkg/util/sets"
|
||||||
"k8s.io/apimachinery/pkg/util/wait"
|
"k8s.io/apimachinery/pkg/util/wait"
|
||||||
"k8s.io/kubernetes/pkg/api/v1/service"
|
"k8s.io/kubernetes/pkg/api/v1/service"
|
||||||
"k8s.io/kubernetes/pkg/cloudprovider"
|
"k8s.io/kubernetes/pkg/cloudprovider"
|
||||||
@ -615,6 +617,29 @@ func getSubnetIDForLB(compute *gophercloud.ServiceClient, node v1.Node) (string,
|
|||||||
return "", ErrNotFound
|
return "", ErrNotFound
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// getNodeSecurityGroupIDForLB lists node-security-groups for specific nodes
|
||||||
|
func getNodeSecurityGroupIDForLB(compute *gophercloud.ServiceClient, nodes []*v1.Node) ([]string, error) {
|
||||||
|
nodeSecurityGroupIDs := sets.NewString()
|
||||||
|
|
||||||
|
for _, node := range nodes {
|
||||||
|
nodeName := types.NodeName(node.Name)
|
||||||
|
srv, err := getServerByName(compute, nodeName)
|
||||||
|
if err != nil {
|
||||||
|
return nodeSecurityGroupIDs.List(), err
|
||||||
|
}
|
||||||
|
|
||||||
|
// use the first node-security-groups
|
||||||
|
// case 0: node1:SG1 node2:SG1 return SG1
|
||||||
|
// case 1: node1:SG1 node2:SG2 return SG1,SG2
|
||||||
|
// case 2: node1:SG1,SG2 node2:SG3,SG4 return SG1,SG3
|
||||||
|
// case 3: node1:SG1,SG2 node2:SG2,SG3 return SG1,SG2
|
||||||
|
securityGroupName := srv.SecurityGroups[0]["name"]
|
||||||
|
nodeSecurityGroupIDs.Insert(securityGroupName.(string))
|
||||||
|
}
|
||||||
|
|
||||||
|
return nodeSecurityGroupIDs.List(), nil
|
||||||
|
}
|
||||||
|
|
||||||
// TODO: This code currently ignores 'region' and always creates a
|
// TODO: This code currently ignores 'region' and always creates a
|
||||||
// loadbalancer in only the current OpenStack region. We should take
|
// loadbalancer in only the current OpenStack region. We should take
|
||||||
// a list of regions (from config) and query/create loadbalancers in
|
// a list of regions (from config) and query/create loadbalancers in
|
||||||
@ -675,7 +700,7 @@ func (lbaas *LbaasV2) EnsureLoadBalancer(clusterName string, apiService *v1.Serv
|
|||||||
|
|
||||||
sourceRanges, err := service.GetLoadBalancerSourceRanges(apiService)
|
sourceRanges, err := service.GetLoadBalancerSourceRanges(apiService)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, fmt.Errorf("Failed to get source ranges for loadbalancer service %s/%s: %v", apiService.Namespace, apiService.Name, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if !service.IsAllowAll(sourceRanges) && !lbaas.opts.ManageSecurityGroups {
|
if !service.IsAllowAll(sourceRanges) && !lbaas.opts.ManageSecurityGroups {
|
||||||
@ -917,30 +942,76 @@ func (lbaas *LbaasV2) EnsureLoadBalancer(clusterName string, apiService *v1.Serv
|
|||||||
}
|
}
|
||||||
|
|
||||||
if lbaas.opts.ManageSecurityGroups {
|
if lbaas.opts.ManageSecurityGroups {
|
||||||
lbSecGroupCreateOpts := groups.CreateOpts{
|
err := lbaas.ensureSecurityGroup(clusterName, apiService, nodes, loadbalancer)
|
||||||
Name: getSecurityGroupName(clusterName, apiService),
|
|
||||||
Description: fmt.Sprintf("Securty Group for %v Service LoadBalancer", apiService.Name),
|
|
||||||
}
|
|
||||||
|
|
||||||
lbSecGroup, err := groups.Create(lbaas.network, lbSecGroupCreateOpts).Extract()
|
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
// cleanup what was created so far
|
// cleanup what was created so far
|
||||||
_ = lbaas.EnsureLoadBalancerDeleted(clusterName, apiService)
|
_ = lbaas.EnsureLoadBalancerDeleted(clusterName, apiService)
|
||||||
return nil, err
|
return status, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return status, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ensureSecurityGroup ensures security group exist for specific loadbalancer service.
|
||||||
|
// Creating security group for specific loadbalancer service when it does not exist.
|
||||||
|
func (lbaas *LbaasV2) ensureSecurityGroup(clusterName string, apiService *v1.Service, nodes []*v1.Node, loadbalancer *loadbalancers.LoadBalancer) error {
|
||||||
|
// find node-security-group for service
|
||||||
|
var err error
|
||||||
|
if len(lbaas.opts.NodeSecurityGroupIDs) == 0 {
|
||||||
|
lbaas.opts.NodeSecurityGroupIDs, err = getNodeSecurityGroupIDForLB(lbaas.compute, nodes)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("Failed to find node-security-group for loadbalancer service %s/%s: %v", apiService.Namespace, apiService.Name, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
glog.V(4).Infof("find node-security-group %v for loadbalancer service %s/%s", lbaas.opts.NodeSecurityGroupIDs, apiService.Namespace, apiService.Name)
|
||||||
|
|
||||||
|
// get service ports
|
||||||
|
ports := apiService.Spec.Ports
|
||||||
|
if len(ports) == 0 {
|
||||||
|
return fmt.Errorf("no ports provided to openstack load balancer")
|
||||||
|
}
|
||||||
|
|
||||||
|
// get service source ranges
|
||||||
|
sourceRanges, err := service.GetLoadBalancerSourceRanges(apiService)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("Failed to get source ranges for loadbalancer service %s/%s: %v", apiService.Namespace, apiService.Name, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ensure security group for LB
|
||||||
|
lbSecGroupName := getSecurityGroupName(clusterName, apiService)
|
||||||
|
lbSecGroupID, err := groups.IDFromName(lbaas.network, lbSecGroupName)
|
||||||
|
if err != nil {
|
||||||
|
// check whether security group does not exist
|
||||||
|
_, ok := err.(*gophercloud.ErrResourceNotFound)
|
||||||
|
if ok {
|
||||||
|
// create it later
|
||||||
|
lbSecGroupID = ""
|
||||||
|
} else {
|
||||||
|
return fmt.Errorf("Error occurred finding security group: %s: %v", lbSecGroupName, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(lbSecGroupID) == 0 {
|
||||||
|
// create security group
|
||||||
|
lbSecGroupCreateOpts := groups.CreateOpts{
|
||||||
|
Name: getSecurityGroupName(clusterName, apiService),
|
||||||
|
Description: fmt.Sprintf("Securty Group for loadbalancer service %s/%s", apiService.Namespace, apiService.Name),
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, port := range ports {
|
lbSecGroup, err := groups.Create(lbaas.network, lbSecGroupCreateOpts).Extract()
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("Failed to create Security Group for loadbalancer service %s/%s: %v", apiService.Namespace, apiService.Name, err)
|
||||||
|
}
|
||||||
|
lbSecGroupID = lbSecGroup.ID
|
||||||
|
|
||||||
|
//add rule in security group
|
||||||
|
for _, port := range ports {
|
||||||
for _, sourceRange := range sourceRanges.StringSlice() {
|
for _, sourceRange := range sourceRanges.StringSlice() {
|
||||||
ethertype := rules.EtherType4
|
ethertype := rules.EtherType4
|
||||||
network, _, err := net.ParseCIDR(sourceRange)
|
network, _, err := net.ParseCIDR(sourceRange)
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
// cleanup what was created so far
|
return fmt.Errorf("Error parsing source range %s as a CIDR: %v", sourceRange, err)
|
||||||
glog.Errorf("Error parsing source range %s as a CIDR", sourceRange)
|
|
||||||
_ = lbaas.EnsureLoadBalancerDeleted(clusterName, apiService)
|
|
||||||
return nil, err
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if network.To4() == nil {
|
if network.To4() == nil {
|
||||||
@ -960,18 +1031,9 @@ func (lbaas *LbaasV2) EnsureLoadBalancer(clusterName string, apiService *v1.Serv
|
|||||||
_, err = rules.Create(lbaas.network, lbSecGroupRuleCreateOpts).Extract()
|
_, err = rules.Create(lbaas.network, lbSecGroupRuleCreateOpts).Extract()
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
// cleanup what was created so far
|
return fmt.Errorf("Error occured creating rule for SecGroup %s: %v", lbSecGroup.ID, err)
|
||||||
_ = lbaas.EnsureLoadBalancerDeleted(clusterName, apiService)
|
|
||||||
return nil, err
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
err := createNodeSecurityGroup(lbaas.network, lbaas.opts.NodeSecurityGroupID, int(port.NodePort), port.Protocol, lbSecGroup.ID)
|
|
||||||
if err != nil {
|
|
||||||
glog.Errorf("Error occured creating security group for loadbalancer %s:", loadbalancer.ID)
|
|
||||||
_ = lbaas.EnsureLoadBalancerDeleted(clusterName, apiService)
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
lbSecGroupRuleCreateOpts := rules.CreateOpts{
|
lbSecGroupRuleCreateOpts := rules.CreateOpts{
|
||||||
@ -987,9 +1049,7 @@ func (lbaas *LbaasV2) EnsureLoadBalancer(clusterName string, apiService *v1.Serv
|
|||||||
_, err = rules.Create(lbaas.network, lbSecGroupRuleCreateOpts).Extract()
|
_, err = rules.Create(lbaas.network, lbSecGroupRuleCreateOpts).Extract()
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
// cleanup what was created so far
|
return fmt.Errorf("Error occured creating rule for SecGroup %s: %v", lbSecGroup.ID, err)
|
||||||
_ = lbaas.EnsureLoadBalancerDeleted(clusterName, apiService)
|
|
||||||
return nil, err
|
|
||||||
}
|
}
|
||||||
|
|
||||||
lbSecGroupRuleCreateOpts = rules.CreateOpts{
|
lbSecGroupRuleCreateOpts = rules.CreateOpts{
|
||||||
@ -1003,25 +1063,68 @@ func (lbaas *LbaasV2) EnsureLoadBalancer(clusterName string, apiService *v1.Serv
|
|||||||
}
|
}
|
||||||
|
|
||||||
_, err = rules.Create(lbaas.network, lbSecGroupRuleCreateOpts).Extract()
|
_, err = rules.Create(lbaas.network, lbSecGroupRuleCreateOpts).Extract()
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
// cleanup what was created so far
|
return fmt.Errorf("Error occured creating rule for SecGroup %s: %v", lbSecGroup.ID, err)
|
||||||
_ = lbaas.EnsureLoadBalancerDeleted(clusterName, apiService)
|
|
||||||
return nil, err
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// get security groups of port
|
||||||
portID := loadbalancer.VipPortID
|
portID := loadbalancer.VipPortID
|
||||||
update_opts := neutronports.UpdateOpts{SecurityGroups: &[]string{lbSecGroup.ID}}
|
port, err := getPortByID(lbaas.network, portID)
|
||||||
res := neutronports.Update(lbaas.network, portID, update_opts)
|
if err != nil {
|
||||||
if res.Err != nil {
|
return err
|
||||||
glog.Errorf("Error occured updating port: %s", portID)
|
}
|
||||||
// cleanup what was created so far
|
|
||||||
_ = lbaas.EnsureLoadBalancerDeleted(clusterName, apiService)
|
// ensure the vip port has the security groups
|
||||||
return nil, res.Err
|
found := false
|
||||||
|
for _, portSecurityGroups := range port.SecurityGroups {
|
||||||
|
if portSecurityGroups == lbSecGroup.ID {
|
||||||
|
found = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// update loadbalancer vip port
|
||||||
|
if !found {
|
||||||
|
port.SecurityGroups = append(port.SecurityGroups, lbSecGroup.ID)
|
||||||
|
update_opts := neutronports.UpdateOpts{SecurityGroups: &port.SecurityGroups}
|
||||||
|
res := neutronports.Update(lbaas.network, portID, update_opts)
|
||||||
|
if res.Err != nil {
|
||||||
|
msg := fmt.Sprintf("Error occured updating port %s for loadbalancer service %s/%s: %v", portID, apiService.Namespace, apiService.Name, res.Err)
|
||||||
|
return fmt.Errorf(msg)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return status, nil
|
// ensure rules for every node security group
|
||||||
|
for _, port := range ports {
|
||||||
|
for _, nodeSecurityGroupID := range lbaas.opts.NodeSecurityGroupIDs {
|
||||||
|
opts := rules.ListOpts{
|
||||||
|
Direction: string(rules.DirIngress),
|
||||||
|
SecGroupID: nodeSecurityGroupID,
|
||||||
|
RemoteGroupID: lbSecGroupID,
|
||||||
|
PortRangeMax: int(port.NodePort),
|
||||||
|
PortRangeMin: int(port.NodePort),
|
||||||
|
Protocol: string(port.Protocol),
|
||||||
|
}
|
||||||
|
secGroupRules, err := getSecurityGroupRules(lbaas.network, opts)
|
||||||
|
if err != nil && !isNotFound(err) {
|
||||||
|
msg := fmt.Sprintf("Error finding rules for remote group id %s in security group id %s: %v", lbSecGroupID, nodeSecurityGroupID, err)
|
||||||
|
return fmt.Errorf(msg)
|
||||||
|
}
|
||||||
|
if len(secGroupRules) != 0 {
|
||||||
|
// Do not add rule when find rules for remote group in the Node Security Group
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
// Add the rules in the Node Security Group
|
||||||
|
err = createNodeSecurityGroup(lbaas.network, nodeSecurityGroupID, int(port.NodePort), port.Protocol, lbSecGroupID)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("Error occured creating security group for loadbalancer service %s/%s: %v", apiService.Namespace, apiService.Name, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (lbaas *LbaasV2) UpdateLoadBalancer(clusterName string, service *v1.Service, nodes []*v1.Node) error {
|
func (lbaas *LbaasV2) UpdateLoadBalancer(clusterName string, service *v1.Service, nodes []*v1.Node) error {
|
||||||
@ -1147,6 +1250,96 @@ func (lbaas *LbaasV2) UpdateLoadBalancer(clusterName string, service *v1.Service
|
|||||||
waitLoadbalancerActiveProvisioningStatus(lbaas.network, loadbalancer.ID)
|
waitLoadbalancerActiveProvisioningStatus(lbaas.network, loadbalancer.ID)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if lbaas.opts.ManageSecurityGroups {
|
||||||
|
err := lbaas.updateSecurityGroup(clusterName, service, nodes, loadbalancer)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("Failed to update Securty Group for loadbalancer service %s/%s: %v", service.Namespace, service.Name, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// updateSecurityGroup updating security group for specific loadbalancer service.
|
||||||
|
func (lbaas *LbaasV2) updateSecurityGroup(clusterName string, apiService *v1.Service, nodes []*v1.Node, loadbalancer *loadbalancers.LoadBalancer) error {
|
||||||
|
originalNodeSecurityGroupIDs := lbaas.opts.NodeSecurityGroupIDs
|
||||||
|
|
||||||
|
var err error
|
||||||
|
lbaas.opts.NodeSecurityGroupIDs, err = getNodeSecurityGroupIDForLB(lbaas.compute, nodes)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("Failed to find node-security-group for loadbalancer service %s/%s: %v", apiService.Namespace, apiService.Name, err)
|
||||||
|
}
|
||||||
|
glog.V(4).Infof("find node-security-group %v for loadbalancer service %s/%s", lbaas.opts.NodeSecurityGroupIDs, apiService.Namespace, apiService.Name)
|
||||||
|
|
||||||
|
original := sets.NewString(originalNodeSecurityGroupIDs...)
|
||||||
|
current := sets.NewString(lbaas.opts.NodeSecurityGroupIDs...)
|
||||||
|
removals := original.Difference(current)
|
||||||
|
|
||||||
|
// Generate Name
|
||||||
|
lbSecGroupName := getSecurityGroupName(clusterName, apiService)
|
||||||
|
lbSecGroupID, err := groups.IDFromName(lbaas.network, lbSecGroupName)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("Error occurred finding security group: %s: %v", lbSecGroupName, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
ports := apiService.Spec.Ports
|
||||||
|
if len(ports) == 0 {
|
||||||
|
return fmt.Errorf("no ports provided to openstack load balancer")
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, port := range ports {
|
||||||
|
for removal := range removals {
|
||||||
|
// Delete the rules in the Node Security Group
|
||||||
|
opts := rules.ListOpts{
|
||||||
|
Direction: string(rules.DirIngress),
|
||||||
|
SecGroupID: removal,
|
||||||
|
RemoteGroupID: lbSecGroupID,
|
||||||
|
PortRangeMax: int(port.NodePort),
|
||||||
|
PortRangeMin: int(port.NodePort),
|
||||||
|
Protocol: string(port.Protocol),
|
||||||
|
}
|
||||||
|
secGroupRules, err := getSecurityGroupRules(lbaas.network, opts)
|
||||||
|
if err != nil && !isNotFound(err) {
|
||||||
|
msg := fmt.Sprintf("Error finding rules for remote group id %s in security group id %s: %v", lbSecGroupID, removal, err)
|
||||||
|
return fmt.Errorf(msg)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, rule := range secGroupRules {
|
||||||
|
res := rules.Delete(lbaas.network, rule.ID)
|
||||||
|
if res.Err != nil && !isNotFound(res.Err) {
|
||||||
|
return fmt.Errorf("Error occurred deleting security group rule: %s: %v", rule.ID, res.Err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, nodeSecurityGroupID := range lbaas.opts.NodeSecurityGroupIDs {
|
||||||
|
opts := rules.ListOpts{
|
||||||
|
Direction: string(rules.DirIngress),
|
||||||
|
SecGroupID: nodeSecurityGroupID,
|
||||||
|
RemoteGroupID: lbSecGroupID,
|
||||||
|
PortRangeMax: int(port.NodePort),
|
||||||
|
PortRangeMin: int(port.NodePort),
|
||||||
|
Protocol: string(port.Protocol),
|
||||||
|
}
|
||||||
|
secGroupRules, err := getSecurityGroupRules(lbaas.network, opts)
|
||||||
|
if err != nil && !isNotFound(err) {
|
||||||
|
msg := fmt.Sprintf("Error finding rules for remote group id %s in security group id %s: %v", lbSecGroupID, nodeSecurityGroupID, err)
|
||||||
|
return fmt.Errorf(msg)
|
||||||
|
}
|
||||||
|
if len(secGroupRules) != 0 {
|
||||||
|
// Do not add rule when find rules for remote group in the Node Security Group
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
// Add the rules in the Node Security Group
|
||||||
|
err = createNodeSecurityGroup(lbaas.network, nodeSecurityGroupID, int(port.NodePort), port.Protocol, lbSecGroupID)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("Error occured creating security group for loadbalancer service %s/%s: %v", apiService.Namespace, apiService.Name, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -1261,8 +1454,14 @@ func (lbaas *LbaasV2) EnsureLoadBalancerDeleted(clusterName string, service *v1.
|
|||||||
lbSecGroupName := getSecurityGroupName(clusterName, service)
|
lbSecGroupName := getSecurityGroupName(clusterName, service)
|
||||||
lbSecGroupID, err := groups.IDFromName(lbaas.network, lbSecGroupName)
|
lbSecGroupID, err := groups.IDFromName(lbaas.network, lbSecGroupName)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
glog.V(1).Infof("Error occurred finding security group: %s: %v", lbSecGroupName, err)
|
// check whether security group does not exist
|
||||||
return nil
|
_, ok := err.(*gophercloud.ErrResourceNotFound)
|
||||||
|
if ok {
|
||||||
|
// It is OK when the security group has been deleted by others.
|
||||||
|
return nil
|
||||||
|
} else {
|
||||||
|
return fmt.Errorf("Error occurred finding security group: %s: %v", lbSecGroupName, err)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
lbSecGroup := groups.Delete(lbaas.network, lbSecGroupID)
|
lbSecGroup := groups.Delete(lbaas.network, lbSecGroupID)
|
||||||
@ -1270,22 +1469,32 @@ func (lbaas *LbaasV2) EnsureLoadBalancerDeleted(clusterName string, service *v1.
|
|||||||
return lbSecGroup.Err
|
return lbSecGroup.Err
|
||||||
}
|
}
|
||||||
|
|
||||||
// Delete the rules in the Node Security Group
|
if len(lbaas.opts.NodeSecurityGroupIDs) == 0 {
|
||||||
opts := rules.ListOpts{
|
// Just happen when nodes have not Security Group, or should not happen
|
||||||
SecGroupID: lbaas.opts.NodeSecurityGroupID,
|
// UpdateLoadBalancer and EnsureLoadBalancer can set lbaas.opts.NodeSecurityGroupIDs when it is empty
|
||||||
RemoteGroupID: lbSecGroupID,
|
// And service controller call UpdateLoadBalancer to set lbaas.opts.NodeSecurityGroupIDs when controller manager service is restarted.
|
||||||
}
|
glog.Warningf("Can not find node-security-group from all the nodes of this cluser when delete loadbalancer service %s/%s",
|
||||||
secGroupRules, err := getSecurityGroupRules(lbaas.network, opts)
|
service.Namespace, service.Name)
|
||||||
|
} else {
|
||||||
|
// Delete the rules in the Node Security Group
|
||||||
|
for _, nodeSecurityGroupID := range lbaas.opts.NodeSecurityGroupIDs {
|
||||||
|
opts := rules.ListOpts{
|
||||||
|
SecGroupID: nodeSecurityGroupID,
|
||||||
|
RemoteGroupID: lbSecGroupID,
|
||||||
|
}
|
||||||
|
secGroupRules, err := getSecurityGroupRules(lbaas.network, opts)
|
||||||
|
|
||||||
if err != nil && !isNotFound(err) {
|
if err != nil && !isNotFound(err) {
|
||||||
glog.Errorf("Error finding rules for remote group id %s in security group id %s", lbSecGroupID, lbaas.opts.NodeSecurityGroupID)
|
msg := fmt.Sprintf("Error finding rules for remote group id %s in security group id %s: %v", lbSecGroupID, nodeSecurityGroupID, err)
|
||||||
return err
|
return fmt.Errorf(msg)
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, rule := range secGroupRules {
|
for _, rule := range secGroupRules {
|
||||||
res := rules.Delete(lbaas.network, rule.ID)
|
res := rules.Delete(lbaas.network, rule.ID)
|
||||||
if res.Err != nil && !isNotFound(res.Err) {
|
if res.Err != nil && !isNotFound(res.Err) {
|
||||||
glog.V(1).Infof("Error occurred deleting security group rule: %s: %v", rule.ID, res.Err)
|
return fmt.Errorf("Error occurred deleting security group rule: %s: %v", rule.ID, res.Err)
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -174,7 +174,6 @@ func TestCheckOpenStackOpts(t *testing.T) {
|
|||||||
MonitorTimeout: timeout,
|
MonitorTimeout: timeout,
|
||||||
MonitorMaxRetries: uint(3),
|
MonitorMaxRetries: uint(3),
|
||||||
ManageSecurityGroups: true,
|
ManageSecurityGroups: true,
|
||||||
NodeSecurityGroupID: "b41d28c2-d02f-4e1e-8ffb-23b8e4f5c144",
|
|
||||||
},
|
},
|
||||||
metadataOpts: MetadataOpts{
|
metadataOpts: MetadataOpts{
|
||||||
SearchOrder: configDriveID,
|
SearchOrder: configDriveID,
|
||||||
@ -195,7 +194,6 @@ func TestCheckOpenStackOpts(t *testing.T) {
|
|||||||
MonitorTimeout: timeout,
|
MonitorTimeout: timeout,
|
||||||
MonitorMaxRetries: uint(3),
|
MonitorMaxRetries: uint(3),
|
||||||
ManageSecurityGroups: true,
|
ManageSecurityGroups: true,
|
||||||
NodeSecurityGroupID: "b41d28c2-d02f-4e1e-8ffb-23b8e4f5c144",
|
|
||||||
},
|
},
|
||||||
metadataOpts: MetadataOpts{
|
metadataOpts: MetadataOpts{
|
||||||
SearchOrder: configDriveID,
|
SearchOrder: configDriveID,
|
||||||
@ -214,7 +212,6 @@ func TestCheckOpenStackOpts(t *testing.T) {
|
|||||||
LBMethod: "ROUND_ROBIN",
|
LBMethod: "ROUND_ROBIN",
|
||||||
CreateMonitor: true,
|
CreateMonitor: true,
|
||||||
ManageSecurityGroups: true,
|
ManageSecurityGroups: true,
|
||||||
NodeSecurityGroupID: "b41d28c2-d02f-4e1e-8ffb-23b8e4f5c144",
|
|
||||||
},
|
},
|
||||||
metadataOpts: MetadataOpts{
|
metadataOpts: MetadataOpts{
|
||||||
SearchOrder: configDriveID,
|
SearchOrder: configDriveID,
|
||||||
@ -224,27 +221,6 @@ func TestCheckOpenStackOpts(t *testing.T) {
|
|||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "test4",
|
name: "test4",
|
||||||
openstackOpts: &OpenStack{
|
|
||||||
provider: nil,
|
|
||||||
lbOpts: LoadBalancerOpts{
|
|
||||||
LBVersion: "v2",
|
|
||||||
SubnetId: "6261548e-ffde-4bc7-bd22-59c83578c5ef",
|
|
||||||
FloatingNetworkId: "38b8b5f9-64dc-4424-bf86-679595714786",
|
|
||||||
LBMethod: "ROUND_ROBIN",
|
|
||||||
CreateMonitor: true,
|
|
||||||
MonitorDelay: delay,
|
|
||||||
MonitorTimeout: timeout,
|
|
||||||
MonitorMaxRetries: uint(3),
|
|
||||||
ManageSecurityGroups: true,
|
|
||||||
},
|
|
||||||
metadataOpts: MetadataOpts{
|
|
||||||
SearchOrder: configDriveID,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
expectedError: fmt.Errorf("node-security-group not set in cloud provider config"),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "test5",
|
|
||||||
openstackOpts: &OpenStack{
|
openstackOpts: &OpenStack{
|
||||||
provider: nil,
|
provider: nil,
|
||||||
metadataOpts: MetadataOpts{
|
metadataOpts: MetadataOpts{
|
||||||
@ -254,7 +230,7 @@ func TestCheckOpenStackOpts(t *testing.T) {
|
|||||||
expectedError: fmt.Errorf("Invalid value in section [Metadata] with key `search-order`. Value cannot be empty"),
|
expectedError: fmt.Errorf("Invalid value in section [Metadata] with key `search-order`. Value cannot be empty"),
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "test6",
|
name: "test5",
|
||||||
openstackOpts: &OpenStack{
|
openstackOpts: &OpenStack{
|
||||||
provider: nil,
|
provider: nil,
|
||||||
metadataOpts: MetadataOpts{
|
metadataOpts: MetadataOpts{
|
||||||
@ -264,7 +240,7 @@ func TestCheckOpenStackOpts(t *testing.T) {
|
|||||||
expectedError: fmt.Errorf("Invalid value in section [Metadata] with key `search-order`. Value cannot contain more than 2 elements"),
|
expectedError: fmt.Errorf("Invalid value in section [Metadata] with key `search-order`. Value cannot contain more than 2 elements"),
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "test7",
|
name: "test6",
|
||||||
openstackOpts: &OpenStack{
|
openstackOpts: &OpenStack{
|
||||||
provider: nil,
|
provider: nil,
|
||||||
metadataOpts: MetadataOpts{
|
metadataOpts: MetadataOpts{
|
||||||
|
Loading…
Reference in New Issue
Block a user