Merge pull request #2051 from anguslees/openstack-provider

Openstack provider
This commit is contained in:
Brendan Burns
2014-12-23 09:33:19 -08:00
459 changed files with 32174 additions and 4452 deletions

View File

@@ -21,37 +21,74 @@ import (
"fmt"
"io"
"net"
"net/url"
"regexp"
"time"
"code.google.com/p/gcfg"
"github.com/rackspace/gophercloud"
"github.com/rackspace/gophercloud/openstack"
"github.com/rackspace/gophercloud/openstack/compute/v2/flavors"
"github.com/rackspace/gophercloud/openstack/compute/v2/servers"
"github.com/rackspace/gophercloud/openstack/networking/v2/extensions/lbaas/members"
"github.com/rackspace/gophercloud/openstack/networking/v2/extensions/lbaas/monitors"
"github.com/rackspace/gophercloud/openstack/networking/v2/extensions/lbaas/pools"
"github.com/rackspace/gophercloud/openstack/networking/v2/extensions/lbaas/vips"
"github.com/rackspace/gophercloud/pagination"
"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
"github.com/GoogleCloudPlatform/kubernetes/pkg/cloudprovider"
"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
"github.com/golang/glog"
)
var ErrServerNotFound = errors.New("Server not found")
var ErrMultipleServersFound = errors.New("Multiple servers matched query")
var ErrFlavorNotFound = errors.New("Flavor not found")
var ErrNotFound = errors.New("Failed to find object")
var ErrMultipleResults = errors.New("Multiple results where only one expected")
var ErrNoAddressFound = errors.New("No address found for host")
var ErrAttrNotFound = errors.New("Expected attribute not found")
// encoding.TextUnmarshaler interface for time.Duration
type MyDuration struct {
time.Duration
}
func (d *MyDuration) UnmarshalText(text []byte) error {
res, err := time.ParseDuration(string(text))
if err != nil {
return err
}
d.Duration = res
return nil
}
type LoadBalancerOpts struct {
SubnetId string `gcfg:"subnet-id"` // required
CreateMonitor bool `gcfg:"create-monitor"`
MonitorDelay MyDuration `gcfg:"monitor-delay"`
MonitorTimeout MyDuration `gcfg:"monitor-timeout"`
MonitorMaxRetries uint `gcfg:"monitor-max-retries"`
}
// OpenStack is an implementation of cloud provider Interface for OpenStack.
type OpenStack struct {
provider string
authOpt gophercloud.AuthOptions
provider *gophercloud.ProviderClient
region string
access *gophercloud.Access
lbOpts LoadBalancerOpts
}
type Config struct {
Global struct {
AuthUrl string
Username, Password string
ApiKey string
TenantId, TenantName string
Region string
AuthUrl string `gcfg:"auth-url"`
Username string
UserId string `gcfg:"user-id"`
Password string
ApiKey string `gcfg:"api-key"`
TenantId string `gcfg:"tenant-id"`
TenantName string `gcfg:"tenant-name"`
DomainId string `gcfg:"domain-id"`
DomainName string `gcfg:"domain-name"`
Region string
}
LoadBalancer LoadBalancerOpts
}
func init() {
@@ -66,11 +103,13 @@ func init() {
func (cfg Config) toAuthOptions() gophercloud.AuthOptions {
return gophercloud.AuthOptions{
Username: cfg.Global.Username,
Password: cfg.Global.Password,
ApiKey: cfg.Global.ApiKey,
TenantId: cfg.Global.TenantId,
TenantName: cfg.Global.TenantName,
IdentityEndpoint: cfg.Global.AuthUrl,
Username: cfg.Global.Username,
UserID: cfg.Global.UserId,
Password: cfg.Global.Password,
APIKey: cfg.Global.ApiKey,
TenantID: cfg.Global.TenantId,
TenantName: cfg.Global.TenantName,
// Persistent service, so we need to be able to renew tokens
AllowReauth: true,
@@ -89,133 +128,460 @@ func readConfig(config io.Reader) (Config, error) {
}
func newOpenStack(cfg Config) (*OpenStack, error) {
os := OpenStack{
provider: cfg.Global.AuthUrl,
authOpt: cfg.toAuthOptions(),
region: cfg.Global.Region,
provider, err := openstack.AuthenticatedClient(cfg.toAuthOptions())
if err != nil {
return nil, err
}
access, err := gophercloud.Authenticate(os.provider, os.authOpt)
os.access = access
return &os, err
os := OpenStack{
provider: provider,
region: cfg.Global.Region,
lbOpts: cfg.LoadBalancer,
}
return &os, nil
}
type Instances struct {
servers gophercloud.CloudServersProvider
compute *gophercloud.ServiceClient
flavor_to_resource map[string]*api.NodeResources // keyed by flavor id
}
// Instances returns an implementation of Instances for OpenStack.
func (os *OpenStack) Instances() (cloudprovider.Instances, bool) {
servers, err := gophercloud.ServersApi(os.access, gophercloud.ApiCriteria{
Type: "compute",
UrlChoice: gophercloud.PublicURL,
Region: os.region,
glog.V(2).Info("openstack.Instances() called")
compute, err := openstack.NewComputeV2(os.provider, gophercloud.EndpointOpts{
Region: os.region,
})
if err != nil {
glog.Warningf("Failed to find compute endpoint: %v", err)
return nil, false
}
flavors, err := servers.ListFlavors()
if err != nil {
return nil, false
}
flavor_to_resource := make(map[string]*api.NodeResources, len(flavors))
for _, flavor := range flavors {
rsrc := api.NodeResources{
Capacity: api.ResourceList{
"cpu": util.NewIntOrStringFromInt(flavor.VCpus),
"memory": util.NewIntOrStringFromString(fmt.Sprintf("%dMiB", flavor.Ram)),
"openstack.org/disk": util.NewIntOrStringFromString(fmt.Sprintf("%dGB", flavor.Disk)),
"openstack.org/rxTxFactor": util.NewIntOrStringFromInt(int(flavor.RxTxFactor * 1000)),
"openstack.org/swap": util.NewIntOrStringFromString(fmt.Sprintf("%dMiB", flavor.Swap)),
},
pager := flavors.ListDetail(compute, nil)
flavor_to_resource := make(map[string]*api.NodeResources)
err = pager.EachPage(func(page pagination.Page) (bool, error) {
flavorList, err := flavors.ExtractFlavors(page)
if err != nil {
return false, err
}
flavor_to_resource[flavor.Id] = &rsrc
for _, flavor := range flavorList {
rsrc := api.NodeResources{
Capacity: api.ResourceList{
"cpu": util.NewIntOrStringFromInt(flavor.VCPUs),
"memory": util.NewIntOrStringFromString(fmt.Sprintf("%dMiB", flavor.RAM)),
"openstack.org/disk": util.NewIntOrStringFromString(fmt.Sprintf("%dGB", flavor.Disk)),
"openstack.org/rxTxFactor": util.NewIntOrStringFromInt(int(flavor.RxTxFactor * 1000)),
"openstack.org/swap": util.NewIntOrStringFromString(fmt.Sprintf("%dMiB", flavor.Swap)),
},
}
flavor_to_resource[flavor.ID] = &rsrc
}
return true, nil
})
if err != nil {
glog.Warningf("Failed to find compute flavors: %v", err)
return nil, false
}
return &Instances{servers, flavor_to_resource}, true
glog.V(2).Infof("Found %v compute flavors", len(flavor_to_resource))
glog.V(1).Info("Claiming to support Instances")
return &Instances{compute, flavor_to_resource}, true
}
func (i *Instances) List(name_filter string) ([]string, error) {
filter := url.Values{}
filter.Set("name", name_filter)
filter.Set("status", "ACTIVE")
glog.V(2).Infof("openstack List(%v) called", name_filter)
servers, err := i.servers.ListServersByFilter(filter)
opts := servers.ListOpts{
Name: name_filter,
Status: "ACTIVE",
}
pager := servers.List(i.compute, opts)
ret := make([]string, 0)
err := pager.EachPage(func(page pagination.Page) (bool, error) {
sList, err := servers.ExtractServers(page)
if err != nil {
return false, err
}
for _, server := range sList {
ret = append(ret, server.Name)
}
return true, nil
})
if err != nil {
return nil, err
}
ret := make([]string, len(servers))
for idx, srv := range servers {
ret[idx] = srv.Name
}
glog.V(2).Infof("Found %v entries: %v", len(ret), ret)
return ret, nil
}
func getServerByName(api gophercloud.CloudServersProvider, name string) (*gophercloud.Server, error) {
filter := url.Values{}
filter.Set("name", fmt.Sprintf("^%s$", regexp.QuoteMeta(name)))
filter.Set("status", "ACTIVE")
func getServerByName(client *gophercloud.ServiceClient, name string) (*servers.Server, error) {
opts := servers.ListOpts{
Name: fmt.Sprintf("^%s$", regexp.QuoteMeta(name)),
Status: "ACTIVE",
}
pager := servers.List(client, opts)
servers, err := api.ListServersByFilter(filter)
serverList := make([]servers.Server, 0, 1)
err := pager.EachPage(func(page pagination.Page) (bool, error) {
s, err := servers.ExtractServers(page)
if err != nil {
return false, err
}
serverList = append(serverList, s...)
if len(serverList) > 1 {
return false, ErrMultipleResults
}
return true, nil
})
if err != nil {
return nil, err
}
if len(servers) == 0 {
return nil, ErrServerNotFound
} else if len(servers) > 1 {
return nil, ErrMultipleServersFound
if len(serverList) == 0 {
return nil, ErrNotFound
} else if len(serverList) > 1 {
return nil, ErrMultipleResults
}
return &servers[0], nil
return &serverList[0], nil
}
func (i *Instances) IPAddress(name string) (net.IP, error) {
srv, err := getServerByName(i.servers, name)
func firstAddr(netblob interface{}) string {
// Run-time types for the win :(
list, ok := netblob.([]interface{})
if !ok || len(list) < 1 {
return ""
}
props, ok := list[0].(map[string]interface{})
if !ok {
return ""
}
tmp, ok := props["addr"]
if !ok {
return ""
}
addr, ok := tmp.(string)
if !ok {
return ""
}
return addr
}
func getAddressByName(api *gophercloud.ServiceClient, name string) (string, error) {
srv, err := getServerByName(api, name)
if err != nil {
return nil, err
return "", err
}
var s string
if len(srv.Addresses.Private) > 0 {
s = srv.Addresses.Private[0].Addr
} else if len(srv.Addresses.Public) > 0 {
s = srv.Addresses.Public[0].Addr
} else if srv.AccessIPv4 != "" {
if s == "" {
s = firstAddr(srv.Addresses["private"])
}
if s == "" {
s = firstAddr(srv.Addresses["public"])
}
if s == "" {
s = srv.AccessIPv4
} else {
}
if s == "" {
s = srv.AccessIPv6
}
return net.ParseIP(s), nil
if s == "" {
return "", ErrNoAddressFound
}
return s, nil
}
func (i *Instances) GetNodeResources(name string) (*api.NodeResources, error) {
srv, err := getServerByName(i.servers, name)
func (i *Instances) IPAddress(name string) (net.IP, error) {
glog.V(2).Infof("IPAddress(%v) called", name)
ip, err := getAddressByName(i.compute, name)
if err != nil {
return nil, err
}
rsrc, ok := i.flavor_to_resource[srv.Flavor.Id]
if !ok {
return nil, ErrFlavorNotFound
glog.V(2).Infof("IPAddress(%v) => %v", name, ip)
return net.ParseIP(ip), err
}
func (i *Instances) GetNodeResources(name string) (*api.NodeResources, error) {
glog.V(2).Infof("GetNodeResources(%v) called", name)
srv, err := getServerByName(i.compute, name)
if err != nil {
return nil, err
}
s, ok := srv.Flavor["id"]
if !ok {
return nil, ErrAttrNotFound
}
flavId, ok := s.(string)
if !ok {
return nil, ErrAttrNotFound
}
rsrc, ok := i.flavor_to_resource[flavId]
if !ok {
return nil, ErrNotFound
}
glog.V(2).Infof("GetNodeResources(%v) => %v", name, rsrc)
return rsrc, nil
}
func (aws *OpenStack) Clusters() (cloudprovider.Clusters, bool) {
func (os *OpenStack) Clusters() (cloudprovider.Clusters, bool) {
return nil, false
}
type LoadBalancer struct {
network *gophercloud.ServiceClient
compute *gophercloud.ServiceClient
opts LoadBalancerOpts
}
func (os *OpenStack) TCPLoadBalancer() (cloudprovider.TCPLoadBalancer, bool) {
return nil, false
// TODO: Search for and support Rackspace loadbalancer API, and others.
network, err := openstack.NewNetworkV2(os.provider, gophercloud.EndpointOpts{
Region: os.region,
})
if err != nil {
glog.Warningf("Failed to find neutron endpoint: %v", err)
return nil, false
}
compute, err := openstack.NewComputeV2(os.provider, gophercloud.EndpointOpts{
Region: os.region,
})
if err != nil {
glog.Warningf("Failed to find compute endpoint: %v", err)
return nil, false
}
glog.V(1).Info("Claiming to support TCPLoadBalancer")
return &LoadBalancer{network, compute, os.lbOpts}, true
}
func getVipByName(client *gophercloud.ServiceClient, name string) (*vips.VirtualIP, error) {
opts := vips.ListOpts{
Name: name,
}
pager := vips.List(client, opts)
vipList := make([]vips.VirtualIP, 0, 1)
err := pager.EachPage(func(page pagination.Page) (bool, error) {
v, err := vips.ExtractVIPs(page)
if err != nil {
return false, err
}
vipList = append(vipList, v...)
if len(vipList) > 1 {
return false, ErrMultipleResults
}
return true, nil
})
if err != nil {
return nil, err
}
if len(vipList) == 0 {
return nil, ErrNotFound
} else if len(vipList) > 1 {
return nil, ErrMultipleResults
}
return &vipList[0], nil
}
func (lb *LoadBalancer) TCPLoadBalancerExists(name, region string) (bool, error) {
vip, err := getVipByName(lb.network, name)
if err == ErrNotFound {
return false, nil
}
return vip != nil, err
}
// TODO: This code currently ignores 'region' and always creates a
// loadbalancer in only the current OpenStack region. We should take
// a list of regions (from config) and query/create loadbalancers in
// each region.
func (lb *LoadBalancer) CreateTCPLoadBalancer(name, region string, externalIP net.IP, port int, hosts []string) (net.IP, error) {
glog.V(2).Infof("CreateTCPLoadBalancer(%v, %v, %v, %v, %v)", name, region, externalIP, port, hosts)
pool, err := pools.Create(lb.network, pools.CreateOpts{
Name: name,
Protocol: pools.ProtocolTCP,
SubnetID: lb.opts.SubnetId,
}).Extract()
if err != nil {
return nil, err
}
for _, host := range hosts {
addr, err := getAddressByName(lb.compute, host)
if err != nil {
return nil, err
}
_, err = members.Create(lb.network, members.CreateOpts{
PoolID: pool.ID,
ProtocolPort: port,
Address: addr,
}).Extract()
if err != nil {
pools.Delete(lb.network, pool.ID)
return nil, err
}
}
var mon *monitors.Monitor
if lb.opts.CreateMonitor {
mon, err = monitors.Create(lb.network, monitors.CreateOpts{
Type: monitors.TypeTCP,
Delay: int(lb.opts.MonitorDelay.Duration.Seconds()),
Timeout: int(lb.opts.MonitorTimeout.Duration.Seconds()),
MaxRetries: int(lb.opts.MonitorMaxRetries),
}).Extract()
if err != nil {
pools.Delete(lb.network, pool.ID)
return nil, err
}
_, err = pools.AssociateMonitor(lb.network, pool.ID, mon.ID).Extract()
if err != nil {
monitors.Delete(lb.network, mon.ID)
pools.Delete(lb.network, pool.ID)
return nil, err
}
}
vip, err := vips.Create(lb.network, vips.CreateOpts{
Name: name,
Description: fmt.Sprintf("Kubernetes external service %s", name),
Address: externalIP.String(),
Protocol: "TCP",
ProtocolPort: port,
PoolID: pool.ID,
}).Extract()
if err != nil {
if mon != nil {
monitors.Delete(lb.network, mon.ID)
}
pools.Delete(lb.network, pool.ID)
return nil, err
}
return net.ParseIP(vip.Address), nil
}
func (lb *LoadBalancer) UpdateTCPLoadBalancer(name, region string, hosts []string) error {
glog.V(2).Infof("UpdateTCPLoadBalancer(%v, %v, %v)", name, region, hosts)
vip, err := getVipByName(lb.network, name)
if err != nil {
return err
}
// Set of member (addresses) that _should_ exist
addrs := map[string]bool{}
for _, host := range hosts {
addr, err := getAddressByName(lb.compute, host)
if err != nil {
return err
}
addrs[addr] = true
}
// Iterate over members that _do_ exist
pager := members.List(lb.network, members.ListOpts{PoolID: vip.PoolID})
err = pager.EachPage(func(page pagination.Page) (bool, error) {
memList, err := members.ExtractMembers(page)
if err != nil {
return false, err
}
for _, member := range memList {
if _, found := addrs[member.Address]; found {
// Member already exists
delete(addrs, member.Address)
} else {
// Member needs to be deleted
err = members.Delete(lb.network, member.ID).ExtractErr()
if err != nil {
return false, err
}
}
}
return true, nil
})
if err != nil {
return err
}
// Anything left in addrs is a new member that needs to be added
for addr := range addrs {
_, err := members.Create(lb.network, members.CreateOpts{
PoolID: vip.PoolID,
Address: addr,
ProtocolPort: vip.ProtocolPort,
}).Extract()
if err != nil {
return err
}
}
return nil
}
func (lb *LoadBalancer) DeleteTCPLoadBalancer(name, region string) error {
glog.V(2).Infof("DeleteTCPLoadBalancer(%v, %v)", name, region)
vip, err := getVipByName(lb.network, name)
if err != nil {
return err
}
pool, err := pools.Get(lb.network, vip.PoolID).Extract()
if err != nil {
return err
}
// Have to delete VIP before pool can be deleted
err = vips.Delete(lb.network, vip.ID).ExtractErr()
if err != nil {
return err
}
// Ignore errors for everything following here
for _, monId := range pool.MonitorIDs {
pools.DisassociateMonitor(lb.network, pool.ID, monId)
}
pools.Delete(lb.network, pool.ID)
return nil
}
func (os *OpenStack) Zones() (cloudprovider.Zones, bool) {
return nil, false
glog.V(1).Info("Claiming to support Zones")
return os, true
}
func (os *OpenStack) GetZone() (cloudprovider.Zone, error) {
glog.V(1).Infof("Current zone is %v", os.region)
return cloudprovider.Zone{Region: os.region}, nil
}

View File

@@ -20,6 +20,9 @@ import (
"os"
"strings"
"testing"
"time"
"github.com/rackspace/gophercloud"
)
func TestReadConfig(t *testing.T) {
@@ -30,8 +33,13 @@ func TestReadConfig(t *testing.T) {
cfg, err := readConfig(strings.NewReader(`
[Global]
authurl = http://auth.url
auth-url = http://auth.url
username = user
[LoadBalancer]
create-monitor = yes
monitor-delay = 1m
monitor-timeout = 30s
monitor-max-retries = 3
`))
if err != nil {
t.Fatalf("Should succeed when a valid config is provided: %s", err)
@@ -39,6 +47,19 @@ username = user
if cfg.Global.AuthUrl != "http://auth.url" {
t.Errorf("incorrect authurl: %s", cfg.Global.AuthUrl)
}
if !cfg.LoadBalancer.CreateMonitor {
t.Errorf("incorrect lb.createmonitor: %s", cfg.LoadBalancer.CreateMonitor)
}
if cfg.LoadBalancer.MonitorDelay.Duration != 1*time.Minute {
t.Errorf("incorrect lb.monitordelay: %s", cfg.LoadBalancer.MonitorDelay)
}
if cfg.LoadBalancer.MonitorTimeout.Duration != 30*time.Second {
t.Errorf("incorrect lb.monitortimeout: %s", cfg.LoadBalancer.MonitorTimeout)
}
if cfg.LoadBalancer.MonitorMaxRetries != 3 {
t.Errorf("incorrect lb.monitormaxretries: %s", cfg.LoadBalancer.MonitorMaxRetries)
}
}
func TestToAuthOptions(t *testing.T) {
@@ -56,14 +77,13 @@ func TestToAuthOptions(t *testing.T) {
}
}
// This allows testing against an existing OpenStack install, using the
// standard OS_* OpenStack client environment variables.
// This allows acceptance testing against an existing OpenStack
// install, using the standard OS_* OpenStack client environment
// variables.
// FIXME: it would be better to hermetically test against canned JSON
// requests/responses.
func configFromEnv() (cfg Config, ok bool) {
cfg.Global.AuthUrl = os.Getenv("OS_AUTH_URL")
// gophercloud wants "provider" to point specifically at tokens URL
if !strings.HasSuffix(cfg.Global.AuthUrl, "/tokens") {
cfg.Global.AuthUrl += "/tokens"
}
cfg.Global.TenantId = os.Getenv("OS_TENANT_ID")
// Rax/nova _insists_ that we don't specify both tenant ID and name
@@ -75,11 +95,14 @@ func configFromEnv() (cfg Config, ok bool) {
cfg.Global.Password = os.Getenv("OS_PASSWORD")
cfg.Global.ApiKey = os.Getenv("OS_API_KEY")
cfg.Global.Region = os.Getenv("OS_REGION_NAME")
cfg.Global.DomainId = os.Getenv("OS_DOMAIN_ID")
cfg.Global.DomainName = os.Getenv("OS_DOMAIN_NAME")
ok = (cfg.Global.AuthUrl != "" &&
cfg.Global.Username != "" &&
(cfg.Global.Password != "" || cfg.Global.ApiKey != "") &&
(cfg.Global.TenantId != "" || cfg.Global.TenantName != ""))
(cfg.Global.TenantId != "" || cfg.Global.TenantName != "" ||
cfg.Global.DomainId != "" || cfg.Global.DomainName != ""))
return
}
@@ -133,3 +156,51 @@ func TestInstances(t *testing.T) {
}
t.Logf("Found GetNodeResources(%s) = %s\n", srvs[0], rsrcs)
}
func TestTCPLoadBalancer(t *testing.T) {
cfg, ok := configFromEnv()
if !ok {
t.Skipf("No config found in environment")
}
os, err := newOpenStack(cfg)
if err != nil {
t.Fatalf("Failed to construct/authenticate OpenStack: %s", err)
}
lb, ok := os.TCPLoadBalancer()
if !ok {
t.Fatalf("TCPLoadBalancer() returned false - perhaps your stack doesn't support Neutron?")
}
exists, err := lb.TCPLoadBalancerExists("noexist", "region")
if err != nil {
t.Fatalf("TCPLoadBalancerExists(\"noexist\") returned error: %s", err)
}
if exists {
t.Fatalf("TCPLoadBalancerExists(\"noexist\") returned true")
}
}
func TestZones(t *testing.T) {
os := OpenStack{
provider: &gophercloud.ProviderClient{
IdentityBase: "http://auth.url/",
},
region: "myRegion",
}
z, ok := os.Zones()
if !ok {
t.Fatalf("Zones() returned false")
}
zone, err := z.GetZone()
if err != nil {
t.Fatalf("GetZone() returned error: %s", err)
}
if zone.Region != "myRegion" {
t.Fatalf("GetZone() returned wrong region (%s)", zone.Region)
}
}