Add support for Instances

This commit is contained in:
Sander van Harmelen 2016-08-04 18:10:56 +02:00
parent ea017719e5
commit 04899f2355
6 changed files with 585 additions and 45 deletions

View File

@ -20,11 +20,13 @@ import (
"errors"
"fmt"
"io"
"os"
"path/filepath"
"github.com/golang/glog"
"github.com/kardianos/osext"
"github.com/xanzy/go-cloudstack/cloudstack"
"gopkg.in/gcfg.v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/kubernetes/pkg/cloudprovider"
"k8s.io/kubernetes/pkg/controller"
@ -48,6 +50,7 @@ type CSConfig struct {
// CSCloud is an implementation of Interface for CloudStack.
type CSCloud struct {
client *cloudstack.CloudStackClient
metadata *metadata
projectID string // If non-"", all resources will be created within this project
zone string
}
@ -64,15 +67,14 @@ func init() {
}
func readConfig(config io.Reader) (*CSConfig, error) {
cfg := &CSConfig{}
if config == nil {
err := fmt.Errorf("no cloud provider config given")
return nil, err
return cfg, nil
}
cfg := &CSConfig{}
if err := gcfg.ReadInto(cfg, config); err != nil {
glog.Errorf("Couldn't parse config: %v", err)
return nil, err
return nil, fmt.Errorf("could not parse cloud provider config: %v", err)
}
return cfg, nil
@ -80,9 +82,42 @@ func readConfig(config io.Reader) (*CSConfig, error) {
// newCSCloud creates a new instance of CSCloud.
func newCSCloud(cfg *CSConfig) (*CSCloud, error) {
client := cloudstack.NewAsyncClient(cfg.Global.APIURL, cfg.Global.APIKey, cfg.Global.SecretKey, !cfg.Global.SSLNoVerify)
cs := &CSCloud{
projectID: cfg.Global.ProjectID,
zone: cfg.Global.Zone,
}
return &CSCloud{client, cfg.Global.ProjectID, cfg.Global.Zone}, nil
exe, err := osext.Executable()
if err != nil {
return nil, fmt.Errorf("cloud not find the service executable: %v", err)
}
// When running the kubelet service it's fine to not specify a config file (or only a
// partial config file) as all needed info can be retrieved anonymously using metadata.
if filepath.Base(exe) == "kubelet" || filepath.Base(exe) == "kubelet.exe" {
// In CloudStack your metadata is always served by the DHCP server.
dhcpServer, err := findDHCPServer()
if err == nil {
glog.V(4).Infof("Found metadata server: %v", dhcpServer)
cs.metadata = &metadata{dhcpServer: dhcpServer, zone: cs.zone}
} else {
glog.Errorf("Error searching metadata server: %v", err)
}
}
if cfg.Global.APIURL != "" && cfg.Global.APIKey != "" && cfg.Global.SecretKey != "" {
cs.client = cloudstack.NewAsyncClient(cfg.Global.APIURL, cfg.Global.APIKey, cfg.Global.SecretKey, !cfg.Global.SSLNoVerify)
}
if cs.client == nil {
if cs.metadata != nil {
glog.V(2).Infof("No API URL, key and secret are provided, so only using metadata!")
} else {
return nil, errors.New("no cloud provider config given")
}
}
return cs, nil
}
// Initialize passes a Kubernetes clientBuilder interface to the cloud provider
@ -90,26 +125,54 @@ func (cs *CSCloud) Initialize(clientBuilder controller.ControllerClientBuilder)
// LoadBalancer returns an implementation of LoadBalancer for CloudStack.
func (cs *CSCloud) LoadBalancer() (cloudprovider.LoadBalancer, bool) {
if cs.client == nil {
return nil, false
}
return cs, true
}
// Instances returns an implementation of Instances for CloudStack.
func (cs *CSCloud) Instances() (cloudprovider.Instances, bool) {
return nil, false
if cs.metadata != nil {
return cs.metadata, true
}
if cs.client == nil {
return nil, false
}
return cs, true
}
// Zones returns an implementation of Zones for CloudStack.
func (cs *CSCloud) Zones() (cloudprovider.Zones, bool) {
if cs.metadata != nil {
return cs.metadata, true
}
if cs.client == nil {
return nil, false
}
return cs, true
}
// Clusters returns an implementation of Clusters for CloudStack.
func (cs *CSCloud) Clusters() (cloudprovider.Clusters, bool) {
if cs.client == nil {
return nil, false
}
return nil, false
}
// Routes returns an implementation of Routes for CloudStack.
func (cs *CSCloud) Routes() (cloudprovider.Routes, bool) {
if cs.client == nil {
return nil, false
}
return nil, false
}
@ -130,20 +193,72 @@ func (cs *CSCloud) HasClusterID() bool {
// GetZone returns the Zone containing the region that the program is running in.
func (cs *CSCloud) GetZone() (cloudprovider.Zone, error) {
zone := cloudprovider.Zone{}
if cs.zone == "" {
hostname, err := os.Hostname()
if err != nil {
return zone, fmt.Errorf("failed to get hostname for retrieving the zone: %v", err)
}
instance, count, err := cs.client.VirtualMachine.GetVirtualMachineByName(hostname)
if err != nil {
if count == 0 {
return zone, fmt.Errorf("could not find instance for retrieving the zone: %v", err)
}
return zone, fmt.Errorf("error getting instance for retrieving the zone: %v", err)
}
cs.zone = instance.Zonename
}
glog.V(2).Infof("Current zone is %v", cs.zone)
return cloudprovider.Zone{Region: cs.zone}, nil
zone.FailureDomain = cs.zone
zone.Region = cs.zone
return zone, nil
}
// GetZoneByProviderID implements Zones.GetZoneByProviderID
// This is particularly useful in external cloud providers where the kubelet
// does not initialize node data.
// GetZoneByProviderID returns the Zone, found by using the provider ID.
func (cs *CSCloud) GetZoneByProviderID(providerID string) (cloudprovider.Zone, error) {
return cloudprovider.Zone{}, errors.New("GetZoneByProviderID not implemented")
zone := cloudprovider.Zone{}
instance, count, err := cs.client.VirtualMachine.GetVirtualMachineByID(
providerID,
cloudstack.WithProject(cs.projectID),
)
if err != nil {
if count == 0 {
return zone, fmt.Errorf("could not find node by ID: %v", providerID)
}
return zone, fmt.Errorf("error retrieving zone: %v", err)
}
glog.V(2).Infof("Current zone is %v", cs.zone)
zone.FailureDomain = instance.Zonename
zone.Region = instance.Zonename
return zone, nil
}
// GetZoneByNodeName implements Zones.GetZoneByNodeName
// This is particularly useful in external cloud providers where the kubelet
// does not initialize node data.
// GetZoneByNodeName returns the Zone, found by using the node name.
func (cs *CSCloud) GetZoneByNodeName(nodeName types.NodeName) (cloudprovider.Zone, error) {
return cloudprovider.Zone{}, errors.New("GetZoneByNodeName not imeplemented")
zone := cloudprovider.Zone{}
instance, count, err := cs.client.VirtualMachine.GetVirtualMachineByName(
string(nodeName),
cloudstack.WithProject(cs.projectID),
)
if err != nil {
if count == 0 {
return zone, fmt.Errorf("could not find node: %v", nodeName)
}
return zone, fmt.Errorf("error retrieving zone: %v", err)
}
glog.V(2).Infof("Current zone is %v", cs.zone)
zone.FailureDomain = instance.Zonename
zone.Region = instance.Zonename
return zone, nil
}

View File

@ -0,0 +1,159 @@
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cloudstack
import (
"errors"
"fmt"
"github.com/golang/glog"
"github.com/xanzy/go-cloudstack/cloudstack"
"k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/kubernetes/pkg/cloudprovider"
)
// NodeAddresses returns the addresses of the specified instance.
func (cs *CSCloud) NodeAddresses(name types.NodeName) ([]v1.NodeAddress, error) {
instance, count, err := cs.client.VirtualMachine.GetVirtualMachineByName(
string(name),
cloudstack.WithProject(cs.projectID),
)
if err != nil {
if count == 0 {
return nil, cloudprovider.InstanceNotFound
}
return nil, fmt.Errorf("error retrieving node addresses: %v", err)
}
return cs.nodeAddresses(instance)
}
// NodeAddressesByProviderID returns the addresses of the specified instance.
func (cs *CSCloud) NodeAddressesByProviderID(providerID string) ([]v1.NodeAddress, error) {
instance, count, err := cs.client.VirtualMachine.GetVirtualMachineByID(
providerID,
cloudstack.WithProject(cs.projectID),
)
if err != nil {
if count == 0 {
return nil, cloudprovider.InstanceNotFound
}
return nil, fmt.Errorf("error retrieving node addresses: %v", err)
}
return cs.nodeAddresses(instance)
}
func (cs *CSCloud) nodeAddresses(instance *cloudstack.VirtualMachine) ([]v1.NodeAddress, error) {
if len(instance.Nic) == 0 {
return nil, errors.New("instance does not have an internal IP")
}
addresses := []v1.NodeAddress{
{Type: v1.NodeInternalIP, Address: instance.Nic[0].Ipaddress},
}
if instance.Publicip != "" {
addresses = append(addresses, v1.NodeAddress{Type: v1.NodeExternalIP, Address: instance.Publicip})
} else {
// Since there is no sane way to determine the external IP if the host isn't
// using static NAT, we will just fire a log message and omit the external IP.
glog.V(4).Infof("Could not determine the public IP of host %v (%v)", instance.Name, instance.Id)
}
return addresses, nil
}
// ExternalID returns the cloud provider ID of the specified instance (deprecated).
func (cs *CSCloud) ExternalID(name types.NodeName) (string, error) {
return cs.InstanceID(name)
}
// InstanceID returns the cloud provider ID of the specified instance.
func (cs *CSCloud) InstanceID(name types.NodeName) (string, error) {
instance, count, err := cs.client.VirtualMachine.GetVirtualMachineByName(
string(name),
cloudstack.WithProject(cs.projectID),
)
if err != nil {
if count == 0 {
return "", cloudprovider.InstanceNotFound
}
return "", fmt.Errorf("error retrieving instance ID: %v", err)
}
return instance.Id, nil
}
// InstanceType returns the type of the specified instance.
func (cs *CSCloud) InstanceType(name types.NodeName) (string, error) {
instance, count, err := cs.client.VirtualMachine.GetVirtualMachineByName(
string(name),
cloudstack.WithProject(cs.projectID),
)
if err != nil {
if count == 0 {
return "", cloudprovider.InstanceNotFound
}
return "", fmt.Errorf("error retrieving instance type: %v", err)
}
return instance.Serviceofferingname, nil
}
// InstanceTypeByProviderID returns the type of the specified instance.
func (cs *CSCloud) InstanceTypeByProviderID(providerID string) (string, error) {
instance, count, err := cs.client.VirtualMachine.GetVirtualMachineByID(
providerID,
cloudstack.WithProject(cs.projectID),
)
if err != nil {
if count == 0 {
return "", cloudprovider.InstanceNotFound
}
return "", fmt.Errorf("error retrieving instance type: %v", err)
}
return instance.Serviceofferingname, nil
}
// AddSSHKeyToAllInstances is currently not implemented.
func (cs *CSCloud) AddSSHKeyToAllInstances(user string, keyData []byte) error {
return errors.New("AddSSHKeyToAllInstances not implemented")
}
// CurrentNodeName returns the name of the node we are currently running on.
func (cs *CSCloud) CurrentNodeName(hostname string) (types.NodeName, error) {
return types.NodeName(hostname), nil
}
// InstanceExistsByProviderID returns if the instance still exists.
func (cs *CSCloud) InstanceExistsByProviderID(providerID string) (bool, error) {
_, count, err := cs.client.VirtualMachine.GetVirtualMachineByID(
providerID,
cloudstack.WithProject(cs.projectID),
)
if err != nil {
if count == 0 {
return false, nil
}
return false, fmt.Errorf("error retrieving instance: %v", err)
}
return true, nil
}

View File

@ -30,8 +30,8 @@ const testClusterName = "testCluster"
func TestReadConfig(t *testing.T) {
_, err := readConfig(nil)
if err == nil {
t.Errorf("Should fail when no config is provided: %v", err)
if err != nil {
t.Fatalf("Should not return an error when no config is provided: %v", err)
}
cfg, err := readConfig(strings.NewReader(`
@ -41,7 +41,6 @@ func TestReadConfig(t *testing.T) {
secret-key = a-valid-secret-key
ssl-no-verify = true
project-id = a-valid-project-id
zone = a-valid-zone
`))
if err != nil {
t.Fatalf("Should succeed when a valid config is provided: %v", err)
@ -59,9 +58,6 @@ func TestReadConfig(t *testing.T) {
if !cfg.Global.SSLNoVerify {
t.Errorf("incorrect ssl-no-verify: %t", cfg.Global.SSLNoVerify)
}
if cfg.Global.Zone != "a-valid-zone" {
t.Errorf("incorrect zone: %s", cfg.Global.Zone)
}
}
// This allows acceptance testing against an existing CloudStack environment.
@ -72,7 +68,6 @@ func configFromEnv() (*CSConfig, bool) {
cfg.Global.APIKey = os.Getenv("CS_API_KEY")
cfg.Global.SecretKey = os.Getenv("CS_SECRET_KEY")
cfg.Global.ProjectID = os.Getenv("CS_PROJECT_ID")
cfg.Global.Zone = os.Getenv("CS_ZONE")
// It is save to ignore the error here. If the input cannot be parsed SSLNoVerify
// will still be a bool with its zero value (false) which is the expected default.
@ -120,23 +115,3 @@ func TestLoadBalancer(t *testing.T) {
t.Fatalf("GetLoadBalancer(\"noexist\") returned exists")
}
}
func TestZones(t *testing.T) {
cs := &CSCloud{
zone: "myRegion",
}
z, ok := cs.Zones()
if !ok {
t.Fatalf("Zones() returned false")
}
zone, err := z.GetZone()
if err != nil {
t.Fatalf("GetZone() returned error: %s", err)
}
if zone.Region != "myRegion" {
t.Fatalf("GetZone() returned wrong region (%s)", zone.Region)
}
}

View File

@ -0,0 +1,211 @@
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cloudstack
import (
"errors"
"fmt"
"io/ioutil"
"net"
"net/http"
"github.com/d2g/dhcp4"
"github.com/golang/glog"
"k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/kubernetes/pkg/cloudprovider"
)
type metadata struct {
dhcpServer string
zone string
}
type metadataType string
const (
metadataTypeExternalIP metadataType = "public-ipv4"
metadataTypeInternalIP metadataType = "local-ipv4"
metadataTypeInstanceID metadataType = "instance-id"
metadataTypeInstanceType metadataType = "service-offering"
metadataTypeZone metadataType = "availability-zone"
)
// NodeAddresses returns the addresses of the specified instance.
func (m *metadata) NodeAddresses(name types.NodeName) ([]v1.NodeAddress, error) {
externalIP, err := m.get(metadataTypeExternalIP)
if err != nil {
return nil, fmt.Errorf("could not get external IP: %v", err)
}
internalIP, err := m.get(metadataTypeInternalIP)
if err != nil {
return nil, fmt.Errorf("could not get internal IP: %v", err)
}
return []v1.NodeAddress{
{Type: v1.NodeExternalIP, Address: externalIP},
{Type: v1.NodeInternalIP, Address: internalIP},
}, nil
}
// NodeAddressesByProviderID returns the addresses of the specified instance.
func (m *metadata) NodeAddressesByProviderID(providerID string) ([]v1.NodeAddress, error) {
return nil, errors.New("NodeAddressesByProviderID not implemented")
}
// ExternalID returns the cloud provider ID of the specified instance (deprecated).
func (m *metadata) ExternalID(name types.NodeName) (string, error) {
return m.InstanceID(name)
}
// InstanceID returns the cloud provider ID of the specified instance.
func (m *metadata) InstanceID(name types.NodeName) (string, error) {
instanceID, err := m.get(metadataTypeInstanceID)
if err != nil {
return "", fmt.Errorf("could not get instance ID: %v", err)
}
zone, err := m.get(metadataTypeZone)
if err != nil {
return "", fmt.Errorf("could not get zone: %v", err)
}
return "/" + zone + "/" + instanceID, nil
}
// InstanceType returns the type of the specified instance.
func (m *metadata) InstanceType(name types.NodeName) (string, error) {
instanceType, err := m.get(metadataTypeInstanceType)
if err == nil {
return "", fmt.Errorf("could not get instance type: %v", err)
}
return instanceType, nil
}
// InstanceTypeByProviderID returns the type of the specified instance.
func (m *metadata) InstanceTypeByProviderID(providerID string) (string, error) {
return "", errors.New("InstanceTypeByProviderID not implemented")
}
// AddSSHKeyToAllInstances is currently not implemented.
func (m *metadata) AddSSHKeyToAllInstances(user string, keyData []byte) error {
return errors.New("AddSSHKeyToAllInstances not implemented")
}
// CurrentNodeName returns the name of the node we are currently running on.
func (m *metadata) CurrentNodeName(hostname string) (types.NodeName, error) {
return types.NodeName(hostname), nil
}
// InstanceExistsByProviderID returns if the instance still exists.
func (m *metadata) InstanceExistsByProviderID(providerID string) (bool, error) {
return false, errors.New("InstanceExistsByProviderID not implemented")
}
// GetZone returns the Zone containing the region that the program is running in.
func (m *metadata) GetZone() (cloudprovider.Zone, error) {
zone := cloudprovider.Zone{}
if m.zone == "" {
zoneName, err := m.get(metadataTypeZone)
if err != nil {
return zone, fmt.Errorf("could not get zone: %v", err)
}
m.zone = zoneName
}
glog.V(2).Infof("Current zone is %v", zone)
zone.FailureDomain = m.zone
zone.Region = m.zone
return zone, nil
}
// GetZoneByProviderID returns the Zone, found by using the provider ID.
func (m *metadata) GetZoneByProviderID(providerID string) (cloudprovider.Zone, error) {
return cloudprovider.Zone{}, errors.New("GetZoneByProviderID not implemented")
}
// GetZoneByNodeName returns the Zone, found by using the node name.
func (m *metadata) GetZoneByNodeName(nodeName types.NodeName) (cloudprovider.Zone, error) {
return cloudprovider.Zone{}, errors.New("GetZoneByNodeName not implemented")
}
func (m *metadata) get(mdType metadataType) (string, error) {
url := fmt.Sprintf("http://%s/latest/meta-data/%s", m.dhcpServer, mdType)
resp, err := http.Get(url)
if err != nil {
return "", fmt.Errorf("error reading metadata: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return "", fmt.Errorf("unexpected HTTP status: %d", resp.StatusCode)
}
data, err := ioutil.ReadAll(resp.Body)
if err != nil {
return "", fmt.Errorf("error reading response body: %d", resp.StatusCode)
}
return string(data), nil
}
func findDHCPServer() (string, error) {
nics, err := net.Interfaces()
if err != nil {
return "", fmt.Errorf("could not get interfaces: %v", err)
}
for _, nic := range nics {
if nic.Flags&net.FlagUp == 1 && nic.Flags&net.FlagLoopback == 0 && nic.Flags&net.FlagPointToPoint == 0 {
addrs, err := nic.Addrs()
if err != nil {
return "", fmt.Errorf("error reading IP addresses from interface %v: %v", nic.Name, err)
}
if addrs != nil {
client, err := newDHCPClient(&nic)
if err != nil {
return "", fmt.Errorf("error creating new DHCP client: %v", err)
}
discoverPacket, err := client.SendDiscoverPacket()
if err != nil {
return "", fmt.Errorf("error sending DHCP discover package: %v", err)
}
offerPacket, err := client.GetOffer(&discoverPacket)
if err != nil {
return "", fmt.Errorf("error recieving DHCP offer package: %v", err)
}
offerPacketOptions := offerPacket.ParseOptions()
if ipaddr, ok := offerPacketOptions[dhcp4.OptionServerIdentifier]; ok {
return net.IP(ipaddr).String(), nil
}
}
}
}
return "", errors.New("no server found")
}

View File

@ -0,0 +1,40 @@
// +build linux
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cloudstack
import (
"net"
"time"
"github.com/d2g/dhcp4client"
)
func newDHCPClient(nic *net.Interface) (*dhcp4client.Client, error) {
pktsock, err := dhcp4client.NewPacketSock(nic.Index)
if err != nil {
return nil, err
}
return dhcp4client.New(
dhcp4client.HardwareAddr(nic.HardwareAddr),
dhcp4client.Timeout(2*time.Second),
dhcp4client.Broadcast(false),
dhcp4client.Connection(pktsock),
)
}

View File

@ -0,0 +1,40 @@
// +build !linux
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cloudstack
import (
"net"
"time"
"github.com/d2g/dhcp4client"
)
func newDHCPClient(nic *net.Interface) (*dhcp4client.Client, error) {
inetsock, err := dhcp4client.NewInetSock()
if err != nil {
return nil, err
}
return dhcp4client.New(
dhcp4client.HardwareAddr(nic.HardwareAddr),
dhcp4client.Timeout(2*time.Second),
dhcp4client.Broadcast(false),
dhcp4client.Connection(inetsock),
)
}