github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-08-01 07:47:56 +00:00
Code Issues Projects Releases Wiki Activity

Security Group support for OpenStack Load Balancers

Browse Source 
This allows security groups to be created and attached to the neutron
port that the loadbalancer is using on the subnet.

The security group ID that is assigned to the nodes needs to be
provided, to allow for traffic from the loadbalancer to the nodePort
to be refelected in the rules.

This adds two config items to the LoadBalancer options -

ManageSecurityGroups (bool)
NodeSecurityGroupID  (string)
This commit is contained in:
Graham Hayes 2016-09-06 17:28:09 -07:00 committed by Davide Agnello
parent 2d02feb5ce
commit ac205183d4
2 changed files with 262 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
pkg/cloudprovider/providers/openstack/openstack.go
View File

@ -72,14 +72,16 @@ type LoadBalancer struct {
}
type LoadBalancerOpts struct {
LBVersion string `gcfg:"lb-version"` // overrides autodetection. v1 or v2
SubnetId string `gcfg:"subnet-id"` // required
FloatingNetworkId string `gcfg:"floating-network-id"`
LBMethod string `gcfg:"lb-method"`
CreateMonitor bool `gcfg:"create-monitor"`
MonitorDelay MyDuration `gcfg:"monitor-delay"`
MonitorTimeout MyDuration `gcfg:"monitor-timeout"`
MonitorMaxRetries uint `gcfg:"monitor-max-retries"`
LBVersion string `gcfg:"lb-version"` // overrides autodetection. v1 or v2
SubnetId string `gcfg:"subnet-id"` // required
FloatingNetworkId string `gcfg:"floating-network-id"`
LBMethod string `gcfg:"lb-method"`
CreateMonitor bool `gcfg:"create-monitor"`
MonitorDelay MyDuration `gcfg:"monitor-delay"`
MonitorTimeout MyDuration `gcfg:"monitor-timeout"`
MonitorMaxRetries uint `gcfg:"monitor-max-retries"`
ManageSecurityGroups bool `gcfg:"manage-security-groups"`
NodeSecurityGroupID string `gcfg:"node-security-group"`
}
// OpenStack is an implementation of cloud provider Interface for OpenStack.

275
pkg/cloudprovider/providers/openstack/openstack_loadbalancer.go
View File

@ -17,8 +17,12 @@ limitations under the License.
package openstack
import (
"fmt"
"net"
"strings"
"time"
"github.com/golang/glog"
"github.com/rackspace/gophercloud"
"github.com/rackspace/gophercloud/openstack/networking/v2/extensions"
"github.com/rackspace/gophercloud/openstack/networking/v2/extensions/layer3/floatingips"
@ -30,12 +34,11 @@ import (
"github.com/rackspace/gophercloud/openstack/networking/v2/extensions/lbaas_v2/loadbalancers"
v2_monitors "github.com/rackspace/gophercloud/openstack/networking/v2/extensions/lbaas_v2/monitors"
v2_pools "github.com/rackspace/gophercloud/openstack/networking/v2/extensions/lbaas_v2/pools"
"github.com/rackspace/gophercloud/openstack/networking/v2/extensions/security/groups"
"github.com/rackspace/gophercloud/openstack/networking/v2/extensions/security/rules"
neutron_ports "github.com/rackspace/gophercloud/openstack/networking/v2/ports"
"github.com/rackspace/gophercloud/pagination"
"fmt"
"github.com/golang/glog"
"k8s.io/kubernetes/pkg/api"
"k8s.io/kubernetes/pkg/api/service"
"k8s.io/kubernetes/pkg/cloudprovider"
@ -77,8 +80,9 @@ func networkExtensions(client *gophercloud.ServiceClient) (map[string]bool, erro
return seen, err
}
func getPortIDByIP(client *gophercloud.ServiceClient, ipAddress string) (string, error) {
var portID string
func getPortByIP(client *gophercloud.ServiceClient, ipAddress string) (neutron_ports.Port, error) {
var targetPort neutron_ports.Port
var portFound = false
err := neutron_ports.List(client, neutron_ports.ListOpts{}).EachPage(func(page pagination.Page) (bool, error) {
portList, err := neutron_ports.ExtractPorts(page)
@ -89,7 +93,8 @@ func getPortIDByIP(client *gophercloud.ServiceClient, ipAddress string) (string,
for _, port := range portList {
for _, ip := range port.FixedIPs {
if ip.IPAddress == ipAddress {
portID = port.ID
targetPort = port
portFound = true
return false, nil
}
}
@ -97,8 +102,18 @@ func getPortIDByIP(client *gophercloud.ServiceClient, ipAddress string) (string,
return true, nil
})
if err == nil && !portFound {
err = ErrNotFound
}
return targetPort, err
}
return portID, err
func getPortIDByIP(client *gophercloud.ServiceClient, ipAddress string) (string, error) {
targetPort, err := getPortByIP(client, ipAddress)
if err != nil {
return targetPort.ID, err
}
return targetPort.ID, nil
}
func getFloatingIPByPortID(client *gophercloud.ServiceClient, portID string) (*floatingips.FloatingIP, error) {
@ -396,6 +411,32 @@ func popMember(members []v2_pools.Member, addr string) []v2_pools.Member {
return members
}
func getSecurityGroupName(clusterName string, service *api.Service) string {
return fmt.Sprintf("lb-sg-%s-%v", clusterName, service.Name)
}
func getSecurityGroupRules(client *gophercloud.ServiceClient, opts rules.ListOpts) ([]rules.SecGroupRule, error) {
pager := rules.List(client, opts)
var securityRules []rules.SecGroupRule
err := pager.EachPage(func(page pagination.Page) (bool, error) {
ruleList, err := rules.ExtractRules(page)
if err != nil {
return false, err
}
securityRules = append(securityRules, ruleList...)
return true, nil
})
if err != nil {
return nil, err
}
return securityRules, nil
}
func waitLoadbalancerActiveProvisioningStatus(client *gophercloud.ServiceClient, loadbalancerID string) (string, error) {
start := time.Now().Second()
for {
@ -438,6 +479,41 @@ func waitLoadbalancerDeleted(client *gophercloud.ServiceClient, loadbalancerID s
}
}
func createNodeSecurityGroup(client *gophercloud.ServiceClient, nodeSecurityGroupID string, port int, protocol string, lbSecGroup string) error {
v4NodeSecGroupRuleCreateOpts := rules.CreateOpts{
Direction: "ingress",
PortRangeMax: port,
PortRangeMin: port,
Protocol: strings.ToLower(protocol),
RemoteGroupID: lbSecGroup,
SecGroupID: nodeSecurityGroupID,
EtherType: "IPv4",
}
v6NodeSecGroupRuleCreateOpts := rules.CreateOpts{
Direction: "ingress",
PortRangeMax: port,
PortRangeMin: port,
Protocol: strings.ToLower(protocol),
RemoteGroupID: lbSecGroup,
SecGroupID: nodeSecurityGroupID,
EtherType: "IPv6",
}
_, err := rules.Create(client, v4NodeSecGroupRuleCreateOpts).Extract()
if err != nil {
return err
}
_, err = rules.Create(client, v6NodeSecGroupRuleCreateOpts).Extract()
if err != nil {
return err
}
return nil
}
func (lbaas *LbaasV2) createLoadBalancer(service *api.Service, name string) (*loadbalancers.LoadBalancer, error) {
createOpts := loadbalancers.CreateOpts{
Name: name,
@ -495,7 +571,16 @@ func (lbaas *LbaasV2) EnsureLoadBalancer(clusterName string, apiService *api.Ser
}
}
affinity := api.ServiceAffinityNone //apiService.Spec.SessionAffinity
sourceRanges, err := service.GetLoadBalancerSourceRanges(apiService)
if err != nil {
return nil, err
}
if !service.IsAllowAll(sourceRanges) && !lbaas.opts.ManageSecurityGroups {
return nil, fmt.Errorf("Source range restrictions are not supported for openstack load balancers without managing security groups")
}
affinity := api.ServiceAffinityNone
var persistence *v2_pools.SessionPersistence
switch affinity {
case api.ServiceAffinityNone:
@ -506,15 +591,6 @@ func (lbaas *LbaasV2) EnsureLoadBalancer(clusterName string, apiService *api.Ser
return nil, fmt.Errorf("unsupported load balancer affinity: %v", affinity)
}
sourceRanges, err := service.GetLoadBalancerSourceRanges(apiService)
if err != nil {
return nil, err
}
if !service.IsAllowAll(sourceRanges) {
return nil, fmt.Errorf("Source range restrictions are not supported for openstack load balancers")
}
name := cloudprovider.GetLoadBalancerName(apiService)
loadbalancer, err := getLoadbalancerByName(lbaas.network, name)
if err != nil {
@ -706,27 +782,143 @@ func (lbaas *LbaasV2) EnsureLoadBalancer(clusterName string, apiService *api.Ser
status.Ingress = []api.LoadBalancerIngress{{IP: loadbalancer.VipAddress}}
portID, err := getPortIDByIP(lbaas.network, loadbalancer.VipAddress)
port, err := getPortByIP(lbaas.network, loadbalancer.VipAddress)
if err != nil {
return nil, fmt.Errorf("Error getting port for LB vip %s: %v", loadbalancer.VipAddress, err)
}
floatIP, err := getFloatingIPByPortID(lbaas.network, portID)
floatIP, err := getFloatingIPByPortID(lbaas.network, port.ID)
if err != nil && err != ErrNotFound {
return nil, fmt.Errorf("Error getting floating ip for port %s: %v", portID, err)
return nil, fmt.Errorf("Error getting floating ip for port %s: %v", port.ID, err)
}
if floatIP == nil && lbaas.opts.FloatingNetworkId != "" {
glog.V(4).Infof("Creating floating ip for loadbalancer %s port %s", loadbalancer.ID, portID)
glog.V(4).Infof("Creating floating ip for loadbalancer %s port %s", loadbalancer.ID, port.ID)
floatIPOpts := floatingips.CreateOpts{
FloatingNetworkID: lbaas.opts.FloatingNetworkId,
PortID: portID,
PortID: port.ID,
}
floatIP, err = floatingips.Create(lbaas.network, floatIPOpts).Extract()
if err != nil {
return nil, fmt.Errorf("Error creating LB floatingip %+v: %v", floatIPOpts, err)
}
}
if floatIP != nil {
status.Ingress = append(status.Ingress, api.LoadBalancerIngress{IP: floatIP.FloatingIP})
}
status.Ingress = append(status.Ingress, api.LoadBalancerIngress{IP: floatIP.FloatingIP})
if lbaas.opts.ManageSecurityGroups {
lbSecGroupCreateOpts := groups.CreateOpts{
Name: getSecurityGroupName(clusterName, apiService),
Description: fmt.Sprintf("Securty Group for %v Service LoadBalancer", apiService.Name),
}
lbSecGroup, err := groups.Create(lbaas.network, lbSecGroupCreateOpts).Extract()
if err != nil {
// cleanup what was created so far
_ = lbaas.EnsureLoadBalancerDeleted(clusterName, apiService)
return nil, err
}
for _, port := range ports {
for _, sourceRange := range sourceRanges.StringSlice() {
ethertype := "IPv4"
network, _, err := net.ParseCIDR(sourceRange)
if err != nil {
// cleanup what was created so far
glog.Errorf("Error parsing source range %s as a CIDR", sourceRange)
_ = lbaas.EnsureLoadBalancerDeleted(clusterName, apiService)
return nil, err
}
if network.To4() == nil {
ethertype = "IPv6"
}
lbSecGroupRuleCreateOpts := rules.CreateOpts{
Direction: "ingress",
PortRangeMax: int(port.Port),
PortRangeMin: int(port.Port),
Protocol: strings.ToLower(string(port.Protocol)),
RemoteIPPrefix: sourceRange,
SecGroupID: lbSecGroup.ID,
EtherType: ethertype,
}
_, err = rules.Create(lbaas.network, lbSecGroupRuleCreateOpts).Extract()
if err != nil {
// cleanup what was created so far
_ = lbaas.EnsureLoadBalancerDeleted(clusterName, apiService)
return nil, err
}
}
err := createNodeSecurityGroup(lbaas.network, lbaas.opts.NodeSecurityGroupID, int(port.NodePort), string(port.Protocol), lbSecGroup.ID)
if err != nil {
glog.Errorf("Error occured creating security group for loadbalancer %s:", loadbalancer.ID)
_ = lbaas.EnsureLoadBalancerDeleted(clusterName, apiService)
return nil, err
}
}
lbSecGroupRuleCreateOpts := rules.CreateOpts{
Direction: "ingress",
PortRangeMax: 4, // ICMP: Code - Values for ICMP "Destination Unreachable: Fragmentation Needed and Don't Fragment was Set"
PortRangeMin: 3, // ICMP: Type
Protocol: "icmp",
RemoteIPPrefix: "0.0.0.0/0", // The Fragmentation packet can come from anywhere along the path back to the sourceRange - we need to all this from all
SecGroupID: lbSecGroup.ID,
EtherType: "IPv4",
}
_, err = rules.Create(lbaas.network, lbSecGroupRuleCreateOpts).Extract()
if err != nil {
// cleanup what was created so far
_ = lbaas.EnsureLoadBalancerDeleted(clusterName, apiService)
return nil, err
}
lbSecGroupRuleCreateOpts = rules.CreateOpts{
Direction: "ingress",
PortRangeMax: 0, // ICMP: Code - Values for ICMP "Packet Too Big"
PortRangeMin: 2, // ICMP: Type
Protocol: "icmp",
RemoteIPPrefix: "::/0", // The Fragmentation packet can come from anywhere along the path back to the sourceRange - we need to all this from all
SecGroupID: lbSecGroup.ID,
EtherType: "IPv6",
}
_, err = rules.Create(lbaas.network, lbSecGroupRuleCreateOpts).Extract()
if err != nil {
// cleanup what was created so far
_ = lbaas.EnsureLoadBalancerDeleted(clusterName, apiService)
return nil, err
}
// Get the port ID
port, err := getPortByIP(lbaas.network, loadbalancer.VipAddress)
if err != nil {
// cleanup what was created so far
_ = lbaas.EnsureLoadBalancerDeleted(clusterName, apiService)
return nil, err
}
update_opts := neutron_ports.UpdateOpts{SecurityGroups: []string{lbSecGroup.ID}}
res := neutron_ports.Update(lbaas.network, port.ID, update_opts)
if res.Err != nil {
glog.Errorf("Error occured updating port: %s", port.ID)
// cleanup what was created so far
_ = lbaas.EnsureLoadBalancerDeleted(clusterName, apiService)
return nil, res.Err
}
}
return status, nil
}
@ -753,6 +945,7 @@ func (lbaas *LbaasV2) UpdateLoadBalancer(clusterName string, service *api.Servic
Protocol string
Port int
}
lbListeners := make(map[portKey]listeners.Listener)
err = listeners.List(lbaas.network, listeners.ListOpts{LoadbalancerID: loadbalancer.ID}).EachPage(func(page pagination.Page) (bool, error) {
listenersList, err := listeners.ExtractListeners(page)
@ -994,6 +1187,42 @@ func (lbaas *LbaasV2) EnsureLoadBalancerDeleted(clusterName string, service *api
return err
}
waitLoadbalancerDeleted(lbaas.network, loadbalancer.ID)
// Delete the Security Group
if lbaas.opts.ManageSecurityGroups {
// Generate Name
lbSecGroupName := getSecurityGroupName(clusterName, service)
lbSecGroupID, err := groups.IDFromName(lbaas.network, lbSecGroupName)
if err != nil {
glog.V(1).Infof("Error occurred finding security group: %s: %v", lbSecGroupName, err)
return nil
}
lbSecGroup := groups.Delete(lbaas.network, lbSecGroupID)
if lbSecGroup.Err != nil && !isNotFound(lbSecGroup.Err) {
return lbSecGroup.Err
}
// Delete the rules in the Node Security Group
opts := rules.ListOpts{
SecGroupID: lbaas.opts.NodeSecurityGroupID,
RemoteGroupID: lbSecGroupID,
}
secGroupRules, err := getSecurityGroupRules(lbaas.network, opts)
if err != nil && !isNotFound(err) {
glog.Errorf("Error finding rules for remote group id %s in security group id %s", lbSecGroupID, lbaas.opts.NodeSecurityGroupID)
return err
}
for _, rule := range secGroupRules {
res := rules.Delete(lbaas.network, rule.ID)
if res.Err != nil && !isNotFound(res.Err) {
glog.V(1).Infof("Error occurred deleting security group rule: %s: %v", rule.ID, res.Err)
}
}
}
return nil
}