diff --git a/pkg/cloudprovider/providers/openstack/openstack.go b/pkg/cloudprovider/providers/openstack/openstack.go index 44cfb43b314..b522ae269b2 100644 --- a/pkg/cloudprovider/providers/openstack/openstack.go +++ b/pkg/cloudprovider/providers/openstack/openstack.go @@ -72,14 +72,16 @@ type LoadBalancer struct { } type LoadBalancerOpts struct { - LBVersion string `gcfg:"lb-version"` // overrides autodetection. v1 or v2 - SubnetId string `gcfg:"subnet-id"` // required - FloatingNetworkId string `gcfg:"floating-network-id"` - LBMethod string `gcfg:"lb-method"` - CreateMonitor bool `gcfg:"create-monitor"` - MonitorDelay MyDuration `gcfg:"monitor-delay"` - MonitorTimeout MyDuration `gcfg:"monitor-timeout"` - MonitorMaxRetries uint `gcfg:"monitor-max-retries"` + LBVersion string `gcfg:"lb-version"` // overrides autodetection. v1 or v2 + SubnetId string `gcfg:"subnet-id"` // required + FloatingNetworkId string `gcfg:"floating-network-id"` + LBMethod string `gcfg:"lb-method"` + CreateMonitor bool `gcfg:"create-monitor"` + MonitorDelay MyDuration `gcfg:"monitor-delay"` + MonitorTimeout MyDuration `gcfg:"monitor-timeout"` + MonitorMaxRetries uint `gcfg:"monitor-max-retries"` + ManageSecurityGroups bool `gcfg:"manage-security-groups"` + NodeSecurityGroupID string `gcfg:"node-security-group"` } // OpenStack is an implementation of cloud provider Interface for OpenStack. diff --git a/pkg/cloudprovider/providers/openstack/openstack_loadbalancer.go b/pkg/cloudprovider/providers/openstack/openstack_loadbalancer.go index de996e9e3d7..399e32ee9e6 100644 --- a/pkg/cloudprovider/providers/openstack/openstack_loadbalancer.go +++ b/pkg/cloudprovider/providers/openstack/openstack_loadbalancer.go @@ -17,8 +17,12 @@ limitations under the License. package openstack import ( + "fmt" + "net" + "strings" "time" + "github.com/golang/glog" "github.com/rackspace/gophercloud" "github.com/rackspace/gophercloud/openstack/networking/v2/extensions" "github.com/rackspace/gophercloud/openstack/networking/v2/extensions/layer3/floatingips" @@ -30,12 +34,11 @@ import ( "github.com/rackspace/gophercloud/openstack/networking/v2/extensions/lbaas_v2/loadbalancers" v2_monitors "github.com/rackspace/gophercloud/openstack/networking/v2/extensions/lbaas_v2/monitors" v2_pools "github.com/rackspace/gophercloud/openstack/networking/v2/extensions/lbaas_v2/pools" + "github.com/rackspace/gophercloud/openstack/networking/v2/extensions/security/groups" + "github.com/rackspace/gophercloud/openstack/networking/v2/extensions/security/rules" neutron_ports "github.com/rackspace/gophercloud/openstack/networking/v2/ports" "github.com/rackspace/gophercloud/pagination" - "fmt" - - "github.com/golang/glog" "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/api/service" "k8s.io/kubernetes/pkg/cloudprovider" @@ -77,8 +80,9 @@ func networkExtensions(client *gophercloud.ServiceClient) (map[string]bool, erro return seen, err } -func getPortIDByIP(client *gophercloud.ServiceClient, ipAddress string) (string, error) { - var portID string +func getPortByIP(client *gophercloud.ServiceClient, ipAddress string) (neutron_ports.Port, error) { + var targetPort neutron_ports.Port + var portFound = false err := neutron_ports.List(client, neutron_ports.ListOpts{}).EachPage(func(page pagination.Page) (bool, error) { portList, err := neutron_ports.ExtractPorts(page) @@ -89,7 +93,8 @@ func getPortIDByIP(client *gophercloud.ServiceClient, ipAddress string) (string, for _, port := range portList { for _, ip := range port.FixedIPs { if ip.IPAddress == ipAddress { - portID = port.ID + targetPort = port + portFound = true return false, nil } } @@ -97,8 +102,18 @@ func getPortIDByIP(client *gophercloud.ServiceClient, ipAddress string) (string, return true, nil }) + if err == nil && !portFound { + err = ErrNotFound + } + return targetPort, err +} - return portID, err +func getPortIDByIP(client *gophercloud.ServiceClient, ipAddress string) (string, error) { + targetPort, err := getPortByIP(client, ipAddress) + if err != nil { + return targetPort.ID, err + } + return targetPort.ID, nil } func getFloatingIPByPortID(client *gophercloud.ServiceClient, portID string) (*floatingips.FloatingIP, error) { @@ -396,6 +411,32 @@ func popMember(members []v2_pools.Member, addr string) []v2_pools.Member { return members } +func getSecurityGroupName(clusterName string, service *api.Service) string { + return fmt.Sprintf("lb-sg-%s-%v", clusterName, service.Name) +} + +func getSecurityGroupRules(client *gophercloud.ServiceClient, opts rules.ListOpts) ([]rules.SecGroupRule, error) { + + pager := rules.List(client, opts) + + var securityRules []rules.SecGroupRule + + err := pager.EachPage(func(page pagination.Page) (bool, error) { + ruleList, err := rules.ExtractRules(page) + if err != nil { + return false, err + } + securityRules = append(securityRules, ruleList...) + return true, nil + }) + + if err != nil { + return nil, err + } + + return securityRules, nil +} + func waitLoadbalancerActiveProvisioningStatus(client *gophercloud.ServiceClient, loadbalancerID string) (string, error) { start := time.Now().Second() for { @@ -438,6 +479,41 @@ func waitLoadbalancerDeleted(client *gophercloud.ServiceClient, loadbalancerID s } } +func createNodeSecurityGroup(client *gophercloud.ServiceClient, nodeSecurityGroupID string, port int, protocol string, lbSecGroup string) error { + v4NodeSecGroupRuleCreateOpts := rules.CreateOpts{ + Direction: "ingress", + PortRangeMax: port, + PortRangeMin: port, + Protocol: strings.ToLower(protocol), + RemoteGroupID: lbSecGroup, + SecGroupID: nodeSecurityGroupID, + EtherType: "IPv4", + } + + v6NodeSecGroupRuleCreateOpts := rules.CreateOpts{ + Direction: "ingress", + PortRangeMax: port, + PortRangeMin: port, + Protocol: strings.ToLower(protocol), + RemoteGroupID: lbSecGroup, + SecGroupID: nodeSecurityGroupID, + EtherType: "IPv6", + } + + _, err := rules.Create(client, v4NodeSecGroupRuleCreateOpts).Extract() + + if err != nil { + return err + } + + _, err = rules.Create(client, v6NodeSecGroupRuleCreateOpts).Extract() + + if err != nil { + return err + } + return nil +} + func (lbaas *LbaasV2) createLoadBalancer(service *api.Service, name string) (*loadbalancers.LoadBalancer, error) { createOpts := loadbalancers.CreateOpts{ Name: name, @@ -495,7 +571,16 @@ func (lbaas *LbaasV2) EnsureLoadBalancer(clusterName string, apiService *api.Ser } } - affinity := api.ServiceAffinityNone //apiService.Spec.SessionAffinity + sourceRanges, err := service.GetLoadBalancerSourceRanges(apiService) + if err != nil { + return nil, err + } + + if !service.IsAllowAll(sourceRanges) && !lbaas.opts.ManageSecurityGroups { + return nil, fmt.Errorf("Source range restrictions are not supported for openstack load balancers without managing security groups") + } + + affinity := api.ServiceAffinityNone var persistence *v2_pools.SessionPersistence switch affinity { case api.ServiceAffinityNone: @@ -506,15 +591,6 @@ func (lbaas *LbaasV2) EnsureLoadBalancer(clusterName string, apiService *api.Ser return nil, fmt.Errorf("unsupported load balancer affinity: %v", affinity) } - sourceRanges, err := service.GetLoadBalancerSourceRanges(apiService) - if err != nil { - return nil, err - } - - if !service.IsAllowAll(sourceRanges) { - return nil, fmt.Errorf("Source range restrictions are not supported for openstack load balancers") - } - name := cloudprovider.GetLoadBalancerName(apiService) loadbalancer, err := getLoadbalancerByName(lbaas.network, name) if err != nil { @@ -706,27 +782,143 @@ func (lbaas *LbaasV2) EnsureLoadBalancer(clusterName string, apiService *api.Ser status.Ingress = []api.LoadBalancerIngress{{IP: loadbalancer.VipAddress}} - portID, err := getPortIDByIP(lbaas.network, loadbalancer.VipAddress) + port, err := getPortByIP(lbaas.network, loadbalancer.VipAddress) if err != nil { return nil, fmt.Errorf("Error getting port for LB vip %s: %v", loadbalancer.VipAddress, err) } - floatIP, err := getFloatingIPByPortID(lbaas.network, portID) + floatIP, err := getFloatingIPByPortID(lbaas.network, port.ID) if err != nil && err != ErrNotFound { - return nil, fmt.Errorf("Error getting floating ip for port %s: %v", portID, err) + return nil, fmt.Errorf("Error getting floating ip for port %s: %v", port.ID, err) } if floatIP == nil && lbaas.opts.FloatingNetworkId != "" { - glog.V(4).Infof("Creating floating ip for loadbalancer %s port %s", loadbalancer.ID, portID) + glog.V(4).Infof("Creating floating ip for loadbalancer %s port %s", loadbalancer.ID, port.ID) floatIPOpts := floatingips.CreateOpts{ FloatingNetworkID: lbaas.opts.FloatingNetworkId, - PortID: portID, + PortID: port.ID, } floatIP, err = floatingips.Create(lbaas.network, floatIPOpts).Extract() if err != nil { return nil, fmt.Errorf("Error creating LB floatingip %+v: %v", floatIPOpts, err) } } + if floatIP != nil { + status.Ingress = append(status.Ingress, api.LoadBalancerIngress{IP: floatIP.FloatingIP}) + } - status.Ingress = append(status.Ingress, api.LoadBalancerIngress{IP: floatIP.FloatingIP}) + if lbaas.opts.ManageSecurityGroups { + lbSecGroupCreateOpts := groups.CreateOpts{ + Name: getSecurityGroupName(clusterName, apiService), + Description: fmt.Sprintf("Securty Group for %v Service LoadBalancer", apiService.Name), + } + + lbSecGroup, err := groups.Create(lbaas.network, lbSecGroupCreateOpts).Extract() + + if err != nil { + // cleanup what was created so far + _ = lbaas.EnsureLoadBalancerDeleted(clusterName, apiService) + return nil, err + } + + for _, port := range ports { + + for _, sourceRange := range sourceRanges.StringSlice() { + ethertype := "IPv4" + network, _, err := net.ParseCIDR(sourceRange) + + if err != nil { + // cleanup what was created so far + glog.Errorf("Error parsing source range %s as a CIDR", sourceRange) + _ = lbaas.EnsureLoadBalancerDeleted(clusterName, apiService) + return nil, err + } + + if network.To4() == nil { + ethertype = "IPv6" + } + + lbSecGroupRuleCreateOpts := rules.CreateOpts{ + Direction: "ingress", + PortRangeMax: int(port.Port), + PortRangeMin: int(port.Port), + Protocol: strings.ToLower(string(port.Protocol)), + RemoteIPPrefix: sourceRange, + SecGroupID: lbSecGroup.ID, + EtherType: ethertype, + } + + _, err = rules.Create(lbaas.network, lbSecGroupRuleCreateOpts).Extract() + + if err != nil { + // cleanup what was created so far + _ = lbaas.EnsureLoadBalancerDeleted(clusterName, apiService) + return nil, err + } + } + + err := createNodeSecurityGroup(lbaas.network, lbaas.opts.NodeSecurityGroupID, int(port.NodePort), string(port.Protocol), lbSecGroup.ID) + if err != nil { + glog.Errorf("Error occured creating security group for loadbalancer %s:", loadbalancer.ID) + _ = lbaas.EnsureLoadBalancerDeleted(clusterName, apiService) + return nil, err + } + } + + lbSecGroupRuleCreateOpts := rules.CreateOpts{ + Direction: "ingress", + PortRangeMax: 4, // ICMP: Code - Values for ICMP "Destination Unreachable: Fragmentation Needed and Don't Fragment was Set" + PortRangeMin: 3, // ICMP: Type + Protocol: "icmp", + RemoteIPPrefix: "0.0.0.0/0", // The Fragmentation packet can come from anywhere along the path back to the sourceRange - we need to all this from all + SecGroupID: lbSecGroup.ID, + EtherType: "IPv4", + } + + _, err = rules.Create(lbaas.network, lbSecGroupRuleCreateOpts).Extract() + + if err != nil { + // cleanup what was created so far + _ = lbaas.EnsureLoadBalancerDeleted(clusterName, apiService) + return nil, err + } + + lbSecGroupRuleCreateOpts = rules.CreateOpts{ + Direction: "ingress", + PortRangeMax: 0, // ICMP: Code - Values for ICMP "Packet Too Big" + PortRangeMin: 2, // ICMP: Type + Protocol: "icmp", + RemoteIPPrefix: "::/0", // The Fragmentation packet can come from anywhere along the path back to the sourceRange - we need to all this from all + SecGroupID: lbSecGroup.ID, + EtherType: "IPv6", + } + + _, err = rules.Create(lbaas.network, lbSecGroupRuleCreateOpts).Extract() + + if err != nil { + // cleanup what was created so far + _ = lbaas.EnsureLoadBalancerDeleted(clusterName, apiService) + return nil, err + } + + // Get the port ID + port, err := getPortByIP(lbaas.network, loadbalancer.VipAddress) + if err != nil { + // cleanup what was created so far + _ = lbaas.EnsureLoadBalancerDeleted(clusterName, apiService) + return nil, err + } + + update_opts := neutron_ports.UpdateOpts{SecurityGroups: []string{lbSecGroup.ID}} + + res := neutron_ports.Update(lbaas.network, port.ID, update_opts) + + if res.Err != nil { + glog.Errorf("Error occured updating port: %s", port.ID) + // cleanup what was created so far + _ = lbaas.EnsureLoadBalancerDeleted(clusterName, apiService) + return nil, res.Err + } + + } return status, nil } @@ -753,6 +945,7 @@ func (lbaas *LbaasV2) UpdateLoadBalancer(clusterName string, service *api.Servic Protocol string Port int } + lbListeners := make(map[portKey]listeners.Listener) err = listeners.List(lbaas.network, listeners.ListOpts{LoadbalancerID: loadbalancer.ID}).EachPage(func(page pagination.Page) (bool, error) { listenersList, err := listeners.ExtractListeners(page) @@ -994,6 +1187,42 @@ func (lbaas *LbaasV2) EnsureLoadBalancerDeleted(clusterName string, service *api return err } waitLoadbalancerDeleted(lbaas.network, loadbalancer.ID) + + // Delete the Security Group + if lbaas.opts.ManageSecurityGroups { + // Generate Name + lbSecGroupName := getSecurityGroupName(clusterName, service) + lbSecGroupID, err := groups.IDFromName(lbaas.network, lbSecGroupName) + if err != nil { + glog.V(1).Infof("Error occurred finding security group: %s: %v", lbSecGroupName, err) + return nil + } + + lbSecGroup := groups.Delete(lbaas.network, lbSecGroupID) + if lbSecGroup.Err != nil && !isNotFound(lbSecGroup.Err) { + return lbSecGroup.Err + } + + // Delete the rules in the Node Security Group + opts := rules.ListOpts{ + SecGroupID: lbaas.opts.NodeSecurityGroupID, + RemoteGroupID: lbSecGroupID, + } + secGroupRules, err := getSecurityGroupRules(lbaas.network, opts) + + if err != nil && !isNotFound(err) { + glog.Errorf("Error finding rules for remote group id %s in security group id %s", lbSecGroupID, lbaas.opts.NodeSecurityGroupID) + return err + } + + for _, rule := range secGroupRules { + res := rules.Delete(lbaas.network, rule.ID) + if res.Err != nil && !isNotFound(res.Err) { + glog.V(1).Infof("Error occurred deleting security group rule: %s: %v", rule.ID, res.Err) + } + } + } + return nil }