From ac205183d4ccec7020f846432d04893910e5cbfa Mon Sep 17 00:00:00 2001 From: Graham Hayes Date: Tue, 6 Sep 2016 17:28:09 -0700 Subject: [PATCH] Security Group support for OpenStack Load Balancers This allows security groups to be created and attached to the neutron port that the loadbalancer is using on the subnet. The security group ID that is assigned to the nodes needs to be provided, to allow for traffic from the loadbalancer to the nodePort to be refelected in the rules. This adds two config items to the LoadBalancer options - ManageSecurityGroups (bool) NodeSecurityGroupID (string) --- .../providers/openstack/openstack.go | 18 +- .../openstack/openstack_loadbalancer.go | 275 ++++++++++++++++-- 2 files changed, 262 insertions(+), 31 deletions(-) diff --git a/pkg/cloudprovider/providers/openstack/openstack.go b/pkg/cloudprovider/providers/openstack/openstack.go index 44cfb43b314..b522ae269b2 100644 --- a/pkg/cloudprovider/providers/openstack/openstack.go +++ b/pkg/cloudprovider/providers/openstack/openstack.go @@ -72,14 +72,16 @@ type LoadBalancer struct { } type LoadBalancerOpts struct { - LBVersion string `gcfg:"lb-version"` // overrides autodetection. v1 or v2 - SubnetId string `gcfg:"subnet-id"` // required - FloatingNetworkId string `gcfg:"floating-network-id"` - LBMethod string `gcfg:"lb-method"` - CreateMonitor bool `gcfg:"create-monitor"` - MonitorDelay MyDuration `gcfg:"monitor-delay"` - MonitorTimeout MyDuration `gcfg:"monitor-timeout"` - MonitorMaxRetries uint `gcfg:"monitor-max-retries"` + LBVersion string `gcfg:"lb-version"` // overrides autodetection. v1 or v2 + SubnetId string `gcfg:"subnet-id"` // required + FloatingNetworkId string `gcfg:"floating-network-id"` + LBMethod string `gcfg:"lb-method"` + CreateMonitor bool `gcfg:"create-monitor"` + MonitorDelay MyDuration `gcfg:"monitor-delay"` + MonitorTimeout MyDuration `gcfg:"monitor-timeout"` + MonitorMaxRetries uint `gcfg:"monitor-max-retries"` + ManageSecurityGroups bool `gcfg:"manage-security-groups"` + NodeSecurityGroupID string `gcfg:"node-security-group"` } // OpenStack is an implementation of cloud provider Interface for OpenStack. diff --git a/pkg/cloudprovider/providers/openstack/openstack_loadbalancer.go b/pkg/cloudprovider/providers/openstack/openstack_loadbalancer.go index de996e9e3d7..399e32ee9e6 100644 --- a/pkg/cloudprovider/providers/openstack/openstack_loadbalancer.go +++ b/pkg/cloudprovider/providers/openstack/openstack_loadbalancer.go @@ -17,8 +17,12 @@ limitations under the License. package openstack import ( + "fmt" + "net" + "strings" "time" + "github.com/golang/glog" "github.com/rackspace/gophercloud" "github.com/rackspace/gophercloud/openstack/networking/v2/extensions" "github.com/rackspace/gophercloud/openstack/networking/v2/extensions/layer3/floatingips" @@ -30,12 +34,11 @@ import ( "github.com/rackspace/gophercloud/openstack/networking/v2/extensions/lbaas_v2/loadbalancers" v2_monitors "github.com/rackspace/gophercloud/openstack/networking/v2/extensions/lbaas_v2/monitors" v2_pools "github.com/rackspace/gophercloud/openstack/networking/v2/extensions/lbaas_v2/pools" + "github.com/rackspace/gophercloud/openstack/networking/v2/extensions/security/groups" + "github.com/rackspace/gophercloud/openstack/networking/v2/extensions/security/rules" neutron_ports "github.com/rackspace/gophercloud/openstack/networking/v2/ports" "github.com/rackspace/gophercloud/pagination" - "fmt" - - "github.com/golang/glog" "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/api/service" "k8s.io/kubernetes/pkg/cloudprovider" @@ -77,8 +80,9 @@ func networkExtensions(client *gophercloud.ServiceClient) (map[string]bool, erro return seen, err } -func getPortIDByIP(client *gophercloud.ServiceClient, ipAddress string) (string, error) { - var portID string +func getPortByIP(client *gophercloud.ServiceClient, ipAddress string) (neutron_ports.Port, error) { + var targetPort neutron_ports.Port + var portFound = false err := neutron_ports.List(client, neutron_ports.ListOpts{}).EachPage(func(page pagination.Page) (bool, error) { portList, err := neutron_ports.ExtractPorts(page) @@ -89,7 +93,8 @@ func getPortIDByIP(client *gophercloud.ServiceClient, ipAddress string) (string, for _, port := range portList { for _, ip := range port.FixedIPs { if ip.IPAddress == ipAddress { - portID = port.ID + targetPort = port + portFound = true return false, nil } } @@ -97,8 +102,18 @@ func getPortIDByIP(client *gophercloud.ServiceClient, ipAddress string) (string, return true, nil }) + if err == nil && !portFound { + err = ErrNotFound + } + return targetPort, err +} - return portID, err +func getPortIDByIP(client *gophercloud.ServiceClient, ipAddress string) (string, error) { + targetPort, err := getPortByIP(client, ipAddress) + if err != nil { + return targetPort.ID, err + } + return targetPort.ID, nil } func getFloatingIPByPortID(client *gophercloud.ServiceClient, portID string) (*floatingips.FloatingIP, error) { @@ -396,6 +411,32 @@ func popMember(members []v2_pools.Member, addr string) []v2_pools.Member { return members } +func getSecurityGroupName(clusterName string, service *api.Service) string { + return fmt.Sprintf("lb-sg-%s-%v", clusterName, service.Name) +} + +func getSecurityGroupRules(client *gophercloud.ServiceClient, opts rules.ListOpts) ([]rules.SecGroupRule, error) { + + pager := rules.List(client, opts) + + var securityRules []rules.SecGroupRule + + err := pager.EachPage(func(page pagination.Page) (bool, error) { + ruleList, err := rules.ExtractRules(page) + if err != nil { + return false, err + } + securityRules = append(securityRules, ruleList...) + return true, nil + }) + + if err != nil { + return nil, err + } + + return securityRules, nil +} + func waitLoadbalancerActiveProvisioningStatus(client *gophercloud.ServiceClient, loadbalancerID string) (string, error) { start := time.Now().Second() for { @@ -438,6 +479,41 @@ func waitLoadbalancerDeleted(client *gophercloud.ServiceClient, loadbalancerID s } } +func createNodeSecurityGroup(client *gophercloud.ServiceClient, nodeSecurityGroupID string, port int, protocol string, lbSecGroup string) error { + v4NodeSecGroupRuleCreateOpts := rules.CreateOpts{ + Direction: "ingress", + PortRangeMax: port, + PortRangeMin: port, + Protocol: strings.ToLower(protocol), + RemoteGroupID: lbSecGroup, + SecGroupID: nodeSecurityGroupID, + EtherType: "IPv4", + } + + v6NodeSecGroupRuleCreateOpts := rules.CreateOpts{ + Direction: "ingress", + PortRangeMax: port, + PortRangeMin: port, + Protocol: strings.ToLower(protocol), + RemoteGroupID: lbSecGroup, + SecGroupID: nodeSecurityGroupID, + EtherType: "IPv6", + } + + _, err := rules.Create(client, v4NodeSecGroupRuleCreateOpts).Extract() + + if err != nil { + return err + } + + _, err = rules.Create(client, v6NodeSecGroupRuleCreateOpts).Extract() + + if err != nil { + return err + } + return nil +} + func (lbaas *LbaasV2) createLoadBalancer(service *api.Service, name string) (*loadbalancers.LoadBalancer, error) { createOpts := loadbalancers.CreateOpts{ Name: name, @@ -495,7 +571,16 @@ func (lbaas *LbaasV2) EnsureLoadBalancer(clusterName string, apiService *api.Ser } } - affinity := api.ServiceAffinityNone //apiService.Spec.SessionAffinity + sourceRanges, err := service.GetLoadBalancerSourceRanges(apiService) + if err != nil { + return nil, err + } + + if !service.IsAllowAll(sourceRanges) && !lbaas.opts.ManageSecurityGroups { + return nil, fmt.Errorf("Source range restrictions are not supported for openstack load balancers without managing security groups") + } + + affinity := api.ServiceAffinityNone var persistence *v2_pools.SessionPersistence switch affinity { case api.ServiceAffinityNone: @@ -506,15 +591,6 @@ func (lbaas *LbaasV2) EnsureLoadBalancer(clusterName string, apiService *api.Ser return nil, fmt.Errorf("unsupported load balancer affinity: %v", affinity) } - sourceRanges, err := service.GetLoadBalancerSourceRanges(apiService) - if err != nil { - return nil, err - } - - if !service.IsAllowAll(sourceRanges) { - return nil, fmt.Errorf("Source range restrictions are not supported for openstack load balancers") - } - name := cloudprovider.GetLoadBalancerName(apiService) loadbalancer, err := getLoadbalancerByName(lbaas.network, name) if err != nil { @@ -706,27 +782,143 @@ func (lbaas *LbaasV2) EnsureLoadBalancer(clusterName string, apiService *api.Ser status.Ingress = []api.LoadBalancerIngress{{IP: loadbalancer.VipAddress}} - portID, err := getPortIDByIP(lbaas.network, loadbalancer.VipAddress) + port, err := getPortByIP(lbaas.network, loadbalancer.VipAddress) if err != nil { return nil, fmt.Errorf("Error getting port for LB vip %s: %v", loadbalancer.VipAddress, err) } - floatIP, err := getFloatingIPByPortID(lbaas.network, portID) + floatIP, err := getFloatingIPByPortID(lbaas.network, port.ID) if err != nil && err != ErrNotFound { - return nil, fmt.Errorf("Error getting floating ip for port %s: %v", portID, err) + return nil, fmt.Errorf("Error getting floating ip for port %s: %v", port.ID, err) } if floatIP == nil && lbaas.opts.FloatingNetworkId != "" { - glog.V(4).Infof("Creating floating ip for loadbalancer %s port %s", loadbalancer.ID, portID) + glog.V(4).Infof("Creating floating ip for loadbalancer %s port %s", loadbalancer.ID, port.ID) floatIPOpts := floatingips.CreateOpts{ FloatingNetworkID: lbaas.opts.FloatingNetworkId, - PortID: portID, + PortID: port.ID, } floatIP, err = floatingips.Create(lbaas.network, floatIPOpts).Extract() if err != nil { return nil, fmt.Errorf("Error creating LB floatingip %+v: %v", floatIPOpts, err) } } + if floatIP != nil { + status.Ingress = append(status.Ingress, api.LoadBalancerIngress{IP: floatIP.FloatingIP}) + } - status.Ingress = append(status.Ingress, api.LoadBalancerIngress{IP: floatIP.FloatingIP}) + if lbaas.opts.ManageSecurityGroups { + lbSecGroupCreateOpts := groups.CreateOpts{ + Name: getSecurityGroupName(clusterName, apiService), + Description: fmt.Sprintf("Securty Group for %v Service LoadBalancer", apiService.Name), + } + + lbSecGroup, err := groups.Create(lbaas.network, lbSecGroupCreateOpts).Extract() + + if err != nil { + // cleanup what was created so far + _ = lbaas.EnsureLoadBalancerDeleted(clusterName, apiService) + return nil, err + } + + for _, port := range ports { + + for _, sourceRange := range sourceRanges.StringSlice() { + ethertype := "IPv4" + network, _, err := net.ParseCIDR(sourceRange) + + if err != nil { + // cleanup what was created so far + glog.Errorf("Error parsing source range %s as a CIDR", sourceRange) + _ = lbaas.EnsureLoadBalancerDeleted(clusterName, apiService) + return nil, err + } + + if network.To4() == nil { + ethertype = "IPv6" + } + + lbSecGroupRuleCreateOpts := rules.CreateOpts{ + Direction: "ingress", + PortRangeMax: int(port.Port), + PortRangeMin: int(port.Port), + Protocol: strings.ToLower(string(port.Protocol)), + RemoteIPPrefix: sourceRange, + SecGroupID: lbSecGroup.ID, + EtherType: ethertype, + } + + _, err = rules.Create(lbaas.network, lbSecGroupRuleCreateOpts).Extract() + + if err != nil { + // cleanup what was created so far + _ = lbaas.EnsureLoadBalancerDeleted(clusterName, apiService) + return nil, err + } + } + + err := createNodeSecurityGroup(lbaas.network, lbaas.opts.NodeSecurityGroupID, int(port.NodePort), string(port.Protocol), lbSecGroup.ID) + if err != nil { + glog.Errorf("Error occured creating security group for loadbalancer %s:", loadbalancer.ID) + _ = lbaas.EnsureLoadBalancerDeleted(clusterName, apiService) + return nil, err + } + } + + lbSecGroupRuleCreateOpts := rules.CreateOpts{ + Direction: "ingress", + PortRangeMax: 4, // ICMP: Code - Values for ICMP "Destination Unreachable: Fragmentation Needed and Don't Fragment was Set" + PortRangeMin: 3, // ICMP: Type + Protocol: "icmp", + RemoteIPPrefix: "0.0.0.0/0", // The Fragmentation packet can come from anywhere along the path back to the sourceRange - we need to all this from all + SecGroupID: lbSecGroup.ID, + EtherType: "IPv4", + } + + _, err = rules.Create(lbaas.network, lbSecGroupRuleCreateOpts).Extract() + + if err != nil { + // cleanup what was created so far + _ = lbaas.EnsureLoadBalancerDeleted(clusterName, apiService) + return nil, err + } + + lbSecGroupRuleCreateOpts = rules.CreateOpts{ + Direction: "ingress", + PortRangeMax: 0, // ICMP: Code - Values for ICMP "Packet Too Big" + PortRangeMin: 2, // ICMP: Type + Protocol: "icmp", + RemoteIPPrefix: "::/0", // The Fragmentation packet can come from anywhere along the path back to the sourceRange - we need to all this from all + SecGroupID: lbSecGroup.ID, + EtherType: "IPv6", + } + + _, err = rules.Create(lbaas.network, lbSecGroupRuleCreateOpts).Extract() + + if err != nil { + // cleanup what was created so far + _ = lbaas.EnsureLoadBalancerDeleted(clusterName, apiService) + return nil, err + } + + // Get the port ID + port, err := getPortByIP(lbaas.network, loadbalancer.VipAddress) + if err != nil { + // cleanup what was created so far + _ = lbaas.EnsureLoadBalancerDeleted(clusterName, apiService) + return nil, err + } + + update_opts := neutron_ports.UpdateOpts{SecurityGroups: []string{lbSecGroup.ID}} + + res := neutron_ports.Update(lbaas.network, port.ID, update_opts) + + if res.Err != nil { + glog.Errorf("Error occured updating port: %s", port.ID) + // cleanup what was created so far + _ = lbaas.EnsureLoadBalancerDeleted(clusterName, apiService) + return nil, res.Err + } + + } return status, nil } @@ -753,6 +945,7 @@ func (lbaas *LbaasV2) UpdateLoadBalancer(clusterName string, service *api.Servic Protocol string Port int } + lbListeners := make(map[portKey]listeners.Listener) err = listeners.List(lbaas.network, listeners.ListOpts{LoadbalancerID: loadbalancer.ID}).EachPage(func(page pagination.Page) (bool, error) { listenersList, err := listeners.ExtractListeners(page) @@ -994,6 +1187,42 @@ func (lbaas *LbaasV2) EnsureLoadBalancerDeleted(clusterName string, service *api return err } waitLoadbalancerDeleted(lbaas.network, loadbalancer.ID) + + // Delete the Security Group + if lbaas.opts.ManageSecurityGroups { + // Generate Name + lbSecGroupName := getSecurityGroupName(clusterName, service) + lbSecGroupID, err := groups.IDFromName(lbaas.network, lbSecGroupName) + if err != nil { + glog.V(1).Infof("Error occurred finding security group: %s: %v", lbSecGroupName, err) + return nil + } + + lbSecGroup := groups.Delete(lbaas.network, lbSecGroupID) + if lbSecGroup.Err != nil && !isNotFound(lbSecGroup.Err) { + return lbSecGroup.Err + } + + // Delete the rules in the Node Security Group + opts := rules.ListOpts{ + SecGroupID: lbaas.opts.NodeSecurityGroupID, + RemoteGroupID: lbSecGroupID, + } + secGroupRules, err := getSecurityGroupRules(lbaas.network, opts) + + if err != nil && !isNotFound(err) { + glog.Errorf("Error finding rules for remote group id %s in security group id %s", lbSecGroupID, lbaas.opts.NodeSecurityGroupID) + return err + } + + for _, rule := range secGroupRules { + res := rules.Delete(lbaas.network, rule.ID) + if res.Err != nil && !isNotFound(res.Err) { + glog.V(1).Infof("Error occurred deleting security group rule: %s: %v", rule.ID, res.Err) + } + } + } + return nil }