diff --git a/pkg/hns/endpoint_windows.go b/pkg/hns/endpoint_windows.go index 9dfa41d9..681582c2 100644 --- a/pkg/hns/endpoint_windows.go +++ b/pkg/hns/endpoint_windows.go @@ -24,6 +24,7 @@ import ( "github.com/containernetworking/cni/pkg/types" current "github.com/containernetworking/cni/pkg/types/100" + "github.com/containernetworking/plugins/pkg/errors" ) @@ -40,7 +41,7 @@ type EndpointInfo struct { IpAddress net.IP } -// GetSandboxContainerID returns the sandbox ID of this pod +// GetSandboxContainerID returns the sandbox ID of this pod. func GetSandboxContainerID(containerID string, netNs string) string { if len(netNs) != 0 && netNs != pauseContainerNetNS { splits := strings.SplitN(netNs, ":", 2) @@ -52,7 +53,7 @@ func GetSandboxContainerID(containerID string, netNs string) string { return containerID } -// short function so we know when to return "" for a string +// GetIpString returns the given IP as a string. func GetIpString(ip *net.IP) string { if len(*ip) == 0 { return "" @@ -61,222 +62,136 @@ func GetIpString(ip *net.IP) string { } } -func GenerateHnsEndpoint(epInfo *EndpointInfo, n *NetConf) (*hcsshim.HNSEndpoint, error) { - // run the IPAM plugin and get back the config to apply - hnsEndpoint, err := hcsshim.GetHNSEndpointByName(epInfo.EndpointName) - if err != nil && !hcsshim.IsNotExist(err) { - return nil, errors.Annotatef(err, "failed to get endpoint %q", epInfo.EndpointName) +// GetDefaultDestinationPrefix returns the default destination prefix according to the given IP type. +func GetDefaultDestinationPrefix(ip *net.IP) string { + destinationPrefix := "0.0.0.0/0" + if ip.To4() == nil { + destinationPrefix = "::/0" } - - if hnsEndpoint != nil { - if hnsEndpoint.VirtualNetwork != epInfo.NetworkId { - _, err = hnsEndpoint.Delete() - if err != nil { - return nil, errors.Annotatef(err, "failed to delete endpoint %s", epInfo.EndpointName) - } - hnsEndpoint = nil - } - } - - if n.LoopbackDSR { - n.ApplyLoopbackDSR(&epInfo.IpAddress) - } - if hnsEndpoint == nil { - hnsEndpoint = &hcsshim.HNSEndpoint{ - Name: epInfo.EndpointName, - VirtualNetwork: epInfo.NetworkId, - DNSServerList: strings.Join(epInfo.DNS.Nameservers, ","), - DNSSuffix: strings.Join(epInfo.DNS.Search, ","), - GatewayAddress: GetIpString(&epInfo.Gateway), - IPAddress: epInfo.IpAddress, - Policies: n.MarshalPolicies(), - } - } - return hnsEndpoint, nil + return destinationPrefix } -func GenerateHcnEndpoint(epInfo *EndpointInfo, n *NetConf) (*hcn.HostComputeEndpoint, error) { - // run the IPAM plugin and get back the config to apply - hcnEndpoint, err := hcn.GetEndpointByName(epInfo.EndpointName) - if err != nil && !hcn.IsNotFoundError(err) { - return nil, errors.Annotatef(err, "failed to get endpoint %q", epInfo.EndpointName) - } - - if hcnEndpoint != nil { - // If the endpont already exists, then we should return error unless - // the endpoint is based on a different network then delete - // should that fail return error - if !strings.EqualFold(hcnEndpoint.HostComputeNetwork, epInfo.NetworkId) { - err = hcnEndpoint.Delete() - if err != nil { - return nil, errors.Annotatef(err, "failed to delete endpoint %s", epInfo.EndpointName) - } - } else { - return nil, fmt.Errorf("endpoint %q already exits", epInfo.EndpointName) - } - } - - if hcnEndpoint == nil { - routes := []hcn.Route{ - { - NextHop: GetIpString(&epInfo.Gateway), - DestinationPrefix: GetDefaultDestinationPrefix(&epInfo.Gateway), - }, - } - - hcnDns := hcn.Dns{ - Search: epInfo.DNS.Search, - ServerList: epInfo.DNS.Nameservers, - } - - hcnIpConfig := hcn.IpConfig{ - IpAddress: GetIpString(&epInfo.IpAddress), - } - ipConfigs := []hcn.IpConfig{hcnIpConfig} - - if n.LoopbackDSR { - n.ApplyLoopbackDSR(&epInfo.IpAddress) - } - hcnEndpoint = &hcn.HostComputeEndpoint{ - SchemaVersion: hcn.Version{Major: 2}, - Name: epInfo.EndpointName, - HostComputeNetwork: epInfo.NetworkId, - Dns: hcnDns, - Routes: routes, - IpConfigurations: ipConfigs, - Policies: func() []hcn.EndpointPolicy { - if n.HcnPolicyArgs == nil { - n.HcnPolicyArgs = []hcn.EndpointPolicy{} - } - return n.HcnPolicyArgs - }(), - } - } - return hcnEndpoint, nil -} - -// ConstructEndpointName constructs enpointId which is used to identify an endpoint from HNS -// There is a special consideration for netNs name here, which is required for Windows Server 1709 -// containerID is the Id of the container on which the endpoint is worked on +// ConstructEndpointName constructs endpoint id which is used to identify an endpoint from HNS/HCN. func ConstructEndpointName(containerID string, netNs string, networkName string) string { return GetSandboxContainerID(containerID, netNs) + "_" + networkName } -// DeprovisionEndpoint removes an endpoint from the container by sending a Detach request to HNS -// For shared endpoint, ContainerDetach is used -// for removing the endpoint completely, HotDetachEndpoint is used -func DeprovisionEndpoint(epName string, netns string, containerID string) error { +// GenerateHnsEndpoint generates an HNSEndpoint with given info and config. +func GenerateHnsEndpoint(epInfo *EndpointInfo, n *NetConf) (*hcsshim.HNSEndpoint, error) { + // run the IPAM plugin and get back the config to apply + hnsEndpoint, err := hcsshim.GetHNSEndpointByName(epInfo.EndpointName) + if err != nil && !hcsshim.IsNotExist(err) { + return nil, errors.Annotatef(err, "failed to get HNSEndpoint %s", epInfo.EndpointName) + } + + if hnsEndpoint != nil { + if strings.EqualFold(hnsEndpoint.VirtualNetwork, epInfo.NetworkId) { + return nil, fmt.Errorf("HNSEndpoint %s is already existed", epInfo.EndpointName) + } + // remove endpoint if corrupted + if _, err = hnsEndpoint.Delete(); err != nil { + return nil, errors.Annotatef(err, "failed to delete corrupted HNSEndpoint %s", epInfo.EndpointName) + } + } + + if n.LoopbackDSR { + n.ApplyLoopbackDSRPolicy(&epInfo.IpAddress) + } + hnsEndpoint = &hcsshim.HNSEndpoint{ + Name: epInfo.EndpointName, + VirtualNetwork: epInfo.NetworkId, + DNSServerList: strings.Join(epInfo.DNS.Nameservers, ","), + DNSSuffix: strings.Join(epInfo.DNS.Search, ","), + GatewayAddress: GetIpString(&epInfo.Gateway), + IPAddress: epInfo.IpAddress, + Policies: n.GetHNSEndpointPolicies(), + } + return hnsEndpoint, nil +} + +// RemoveHnsEndpoint detaches the given name endpoint from container specified by containerID, +// or removes the given name endpoint completely. +func RemoveHnsEndpoint(epName string, netns string, containerID string) error { if len(netns) == 0 { return nil } hnsEndpoint, err := hcsshim.GetHNSEndpointByName(epName) - - if hcsshim.IsNotExist(err) { - return nil - } else if err != nil { + if err != nil { + if hcsshim.IsNotExist(err) { + return nil + } return errors.Annotatef(err, "failed to find HNSEndpoint %s", epName) } + // for shared endpoint, detach it from the container if netns != pauseContainerNetNS { - // Shared endpoint removal. Do not remove the endpoint. - hnsEndpoint.ContainerDetach(containerID) + _ = hnsEndpoint.ContainerDetach(containerID) return nil } - // Do not consider this as failure, else this would leak endpoints - hcsshim.HotDetachEndpoint(containerID, hnsEndpoint.Id) - - // Do not return error - hnsEndpoint.Delete() - + // for removing the endpoint completely, hot detach is used at first + _ = hcsshim.HotDetachEndpoint(containerID, hnsEndpoint.Id) + _, _ = hnsEndpoint.Delete() return nil } -type EndpointMakerFunc func() (*hcsshim.HNSEndpoint, error) +type HnsEndpointMakerFunc func() (*hcsshim.HNSEndpoint, error) -// ProvisionEndpoint provisions an endpoint to a container specified by containerID. -// If an endpoint already exists, the endpoint is reused. -// This call is idempotent -func ProvisionEndpoint(epName string, expectedNetworkId string, containerID string, netns string, makeEndpoint EndpointMakerFunc) (*hcsshim.HNSEndpoint, error) { - // On the second add call we expect that the endpoint already exists. If it - // does not then we should return an error. - if netns != pauseContainerNetNS { - _, err := hcsshim.GetHNSEndpointByName(epName) - if err != nil { +// AddHnsEndpoint attaches an HNSEndpoint to a container specified by containerID. +func AddHnsEndpoint(epName string, expectedNetworkId string, containerID string, netns string, makeEndpoint HnsEndpointMakerFunc) (*hcsshim.HNSEndpoint, error) { + hnsEndpoint, err := hcsshim.GetHNSEndpointByName(epName) + if err != nil { + if !hcsshim.IsNotExist(err) { return nil, errors.Annotatef(err, "failed to find HNSEndpoint %s", epName) } } - // check if endpoint already exists - createEndpoint := true - hnsEndpoint, err := hcsshim.GetHNSEndpointByName(epName) - if hnsEndpoint != nil && strings.EqualFold(hnsEndpoint.VirtualNetwork, expectedNetworkId) { - createEndpoint = false + // for shared endpoint, we expect that the endpoint already exists + if netns != pauseContainerNetNS { + if hnsEndpoint == nil { + return nil, errors.Annotatef(err, "failed to find HNSEndpoint %s", epName) + } } - if createEndpoint { - if hnsEndpoint != nil { - if _, err = hnsEndpoint.Delete(); err != nil { - return nil, errors.Annotate(err, "failed to delete the stale HNSEndpoint") + // verify the existing endpoint is corrupted or not + if hnsEndpoint != nil { + if !strings.EqualFold(hnsEndpoint.VirtualNetwork, expectedNetworkId) { + if _, err := hnsEndpoint.Delete(); err != nil { + return nil, errors.Annotatef(err, "failed to delete corrupted HNSEndpoint %s", epName) } + hnsEndpoint = nil } + } + // create endpoint if not found + var isNewEndpoint bool + if hnsEndpoint == nil { if hnsEndpoint, err = makeEndpoint(); err != nil { return nil, errors.Annotate(err, "failed to make a new HNSEndpoint") } - if hnsEndpoint, err = hnsEndpoint.Create(); err != nil { return nil, errors.Annotate(err, "failed to create the new HNSEndpoint") } - + isNewEndpoint = true } - // hot attach + // attach to container if err := hcsshim.HotAttachEndpoint(containerID, hnsEndpoint.Id); err != nil { - if createEndpoint { - err := DeprovisionEndpoint(epName, netns, containerID) - if err != nil { - return nil, errors.Annotatef(err, "failed to Deprovsion after HotAttach failure") + if isNewEndpoint { + if err := RemoveHnsEndpoint(epName, netns, containerID); err != nil { + return nil, errors.Annotatef(err, "failed to remove the new HNSEndpoint %s after attaching container %s failure", hnsEndpoint.Id, containerID) } - } - if hcsshim.ErrComputeSystemDoesNotExist == err { + } else if hcsshim.ErrComputeSystemDoesNotExist == err { return hnsEndpoint, nil } - return nil, err + return nil, errors.Annotatef(err, "failed to attach container %s to HNSEndpoint %s", containerID, hnsEndpoint.Id) } - return hnsEndpoint, nil } -type HcnEndpointMakerFunc func() (*hcn.HostComputeEndpoint, error) - -func AddHcnEndpoint(epName string, expectedNetworkId string, namespace string, - makeEndpoint HcnEndpointMakerFunc) (*hcn.HostComputeEndpoint, error) { - - hcnEndpoint, err := makeEndpoint() - if err != nil { - return nil, errors.Annotate(err, "failed to make a new HNSEndpoint") - } - - if hcnEndpoint, err = hcnEndpoint.Create(); err != nil { - return nil, errors.Annotate(err, "failed to create the new HNSEndpoint") - } - - err = hcn.AddNamespaceEndpoint(namespace, hcnEndpoint.Id) - if err != nil { - err := RemoveHcnEndpoint(epName) - if err != nil { - return nil, errors.Annotatef(err, "failed to Remove Endpoint after AddNamespaceEndpoint failure") - } - return nil, errors.Annotate(err, "failed to Add endpoint to namespace") - } - return hcnEndpoint, nil - -} - -// ConstructResult constructs the CNI result for the endpoint -func ConstructResult(hnsNetwork *hcsshim.HNSNetwork, hnsEndpoint *hcsshim.HNSEndpoint) (*current.Result, error) { +// ConstructHnsResult constructs the CNI result for the HNSEndpoint. +func ConstructHnsResult(hnsNetwork *hcsshim.HNSNetwork, hnsEndpoint *hcsshim.HNSEndpoint) (*current.Result, error) { resultInterface := ¤t.Interface{ Name: hnsEndpoint.Name, Mac: hnsEndpoint.MacAddress, @@ -296,7 +211,7 @@ func ConstructResult(hnsNetwork *hcsshim.HNSNetwork, hnsEndpoint *hcsshim.HNSEnd CNIVersion: current.ImplementedSpecVersion, Interfaces: []*current.Interface{resultInterface}, IPs: []*current.IPConfig{resultIPConfig}, - DNS: types.DNS{ + DNS: types.DNS{ Search: strings.Split(hnsEndpoint.DNSSuffix, ","), Nameservers: strings.Split(hnsEndpoint.DNSServerList, ","), }, @@ -305,24 +220,121 @@ func ConstructResult(hnsNetwork *hcsshim.HNSNetwork, hnsEndpoint *hcsshim.HNSEnd return result, nil } -// This version follows the v2 workflow of removing the endpoint from the namespace and deleting it +// GenerateHcnEndpoint generates a HostComputeEndpoint with given info and config. +func GenerateHcnEndpoint(epInfo *EndpointInfo, n *NetConf) (*hcn.HostComputeEndpoint, error) { + // run the IPAM plugin and get back the config to apply + hcnEndpoint, err := hcn.GetEndpointByName(epInfo.EndpointName) + if err != nil && !hcn.IsNotFoundError(err) { + return nil, errors.Annotatef(err, "failed to get HostComputeEndpoint %s", epInfo.EndpointName) + } + + // verify the existing endpoint is corrupted or not + if hcnEndpoint != nil { + if strings.EqualFold(hcnEndpoint.HostComputeNetwork, epInfo.NetworkId) { + return nil, fmt.Errorf("HostComputeNetwork %s is already existed", epInfo.EndpointName) + } + // remove endpoint if corrupted + if err := hcnEndpoint.Delete(); err != nil { + return nil, errors.Annotatef(err, "failed to delete corrupted HostComputeEndpoint %s", epInfo.EndpointName) + } + } + + if n.LoopbackDSR { + n.ApplyLoopbackDSRPolicy(&epInfo.IpAddress) + } + hcnEndpoint = &hcn.HostComputeEndpoint{ + SchemaVersion: hcn.SchemaVersion{ + Major: 2, + Minor: 0, + }, + Name: epInfo.EndpointName, + HostComputeNetwork: epInfo.NetworkId, + Dns: hcn.Dns{ + Domain: epInfo.DNS.Domain, + Search: epInfo.DNS.Search, + ServerList: epInfo.DNS.Nameservers, + Options: epInfo.DNS.Options, + }, + Routes: []hcn.Route{ + { + NextHop: GetIpString(&epInfo.Gateway), + DestinationPrefix: GetDefaultDestinationPrefix(&epInfo.Gateway), + }, + }, + IpConfigurations: []hcn.IpConfig{ + { + IpAddress: GetIpString(&epInfo.IpAddress), + }, + }, + Policies: n.GetHostComputeEndpointPolicies(), + } + return hcnEndpoint, nil +} + +// RemoveHcnEndpoint removes the given name endpoint from namespace. func RemoveHcnEndpoint(epName string) error { hcnEndpoint, err := hcn.GetEndpointByName(epName) - if hcn.IsNotFoundError(err) { - return nil - } else if err != nil { - _ = fmt.Errorf("[win-cni] Failed to find endpoint %v, err:%v", epName, err) - return err - } - if hcnEndpoint != nil { - err = hcnEndpoint.Delete() - if err != nil { - return fmt.Errorf("[win-cni] Failed to delete endpoint %v, err:%v", epName, err) + if err != nil { + if hcn.IsNotFoundError(err) { + return nil } + return errors.Annotatef(err, "failed to find HostComputeEndpoint %s", epName) + } + + err = hcnEndpoint.Delete() + if err != nil { + return errors.Annotatef(err, "failed to remove HostComputeEndpoint %s", epName) } return nil } +type HcnEndpointMakerFunc func() (*hcn.HostComputeEndpoint, error) + +// AddHcnEndpoint attaches a HostComputeEndpoint to the given namespace. +func AddHcnEndpoint(epName string, expectedNetworkId string, namespace string, makeEndpoint HcnEndpointMakerFunc) (*hcn.HostComputeEndpoint, error) { + hcnEndpoint, err := hcn.GetEndpointByName(epName) + if err != nil { + if !hcn.IsNotFoundError(err) { + return nil, errors.Annotatef(err, "failed to find HostComputeEndpoint %s", epName) + } + } + + // verify the existing endpoint is corrupted or not + if hcnEndpoint != nil { + if !strings.EqualFold(hcnEndpoint.HostComputeNetwork, expectedNetworkId) { + if err := hcnEndpoint.Delete(); err != nil { + return nil, errors.Annotatef(err, "failed to delete corrupted HostComputeEndpoint %s", epName) + } + hcnEndpoint = nil + } + } + + // create endpoint if not found + var isNewEndpoint bool + if hcnEndpoint == nil { + if hcnEndpoint, err = makeEndpoint(); err != nil { + return nil, errors.Annotate(err, "failed to make a new HostComputeEndpoint") + } + if hcnEndpoint, err = hcnEndpoint.Create(); err != nil { + return nil, errors.Annotate(err, "failed to create the new HostComputeEndpoint") + } + isNewEndpoint = true + } + + // add to namespace + err = hcn.AddNamespaceEndpoint(namespace, hcnEndpoint.Id) + if err != nil { + if isNewEndpoint { + if err := RemoveHcnEndpoint(epName); err != nil { + return nil, errors.Annotatef(err, "failed to remove the new HostComputeEndpoint %s after adding HostComputeNamespace %s failure", epName, namespace) + } + } + return nil, errors.Annotatef(err, "failed to add HostComputeEndpoint %s to HostComputeNamespace %s", epName, namespace) + } + return hcnEndpoint, nil +} + +// ConstructHcnResult constructs the CNI result for the HostComputeEndpoint. func ConstructHcnResult(hcnNetwork *hcn.HostComputeNetwork, hcnEndpoint *hcn.HostComputeEndpoint) (*current.Result, error) { resultInterface := ¤t.Interface{ Name: hcnEndpoint.Name, @@ -344,9 +356,11 @@ func ConstructHcnResult(hcnNetwork *hcn.HostComputeNetwork, hcnEndpoint *hcn.Hos CNIVersion: current.ImplementedSpecVersion, Interfaces: []*current.Interface{resultInterface}, IPs: []*current.IPConfig{resultIPConfig}, - DNS: types.DNS{ + DNS: types.DNS{ Search: hcnEndpoint.Dns.Search, Nameservers: hcnEndpoint.Dns.ServerList, + Options: hcnEndpoint.Dns.Options, + Domain: hcnEndpoint.Dns.Domain, }, } diff --git a/pkg/hns/netconf_suite_windows_test.go b/pkg/hns/netconf_suite_windows_test.go index cc69e6fd..90675961 100644 --- a/pkg/hns/netconf_suite_windows_test.go +++ b/pkg/hns/netconf_suite_windows_test.go @@ -20,7 +20,7 @@ import ( "testing" ) -func TestHns(t *testing.T) { +func TestNetConf(t *testing.T) { RegisterFailHandler(Fail) - RunSpecs(t, "HNS NetConf Suite") + RunSpecs(t, "NetConf Suite") } diff --git a/pkg/hns/netconf_windows.go b/pkg/hns/netconf_windows.go index 01974333..cae0589f 100644 --- a/pkg/hns/netconf_windows.go +++ b/pkg/hns/netconf_windows.go @@ -17,9 +17,10 @@ package hns import ( "bytes" "encoding/json" + "errors" "fmt" "net" - + "strconv" "strings" "github.com/Microsoft/hcsshim/hcn" @@ -30,16 +31,16 @@ import ( // NetConf is the CNI spec type NetConf struct { types.NetConf - // ApiVersion is either 1 or 2, which specifies which hns APIs to call - ApiVersion int `json:"ApiVersion"` - // V2 Api Policies - HcnPolicyArgs []hcn.EndpointPolicy `json:"HcnPolicyArgs,omitempty"` - // V1 Api Policies - Policies []policy `json:"policies,omitempty"` - // Options to be passed in by the runtime + // ApiVersion specifies the policies type of HNS or HCN, select one of [1, 2]. + // HNS is the v1 API, which is the default version and applies to dockershim. + // HCN is the v2 API, which can leverage HostComputeNamespace and use in containerd. + ApiVersion int `json:"apiVersion,omitempty"` + // Policies specifies the policy list for HNSEndpoint or HostComputeEndpoint. + Policies []Policy `json:"policies,omitempty"` + // RuntimeConfig represents the options to be passed in by the runtime. RuntimeConfig RuntimeConfig `json:"runtimeConfig"` - // If true, adds a policy to endpoints to support loopback direct server return - LoopbackDSR bool `json:"loopbackDSR"` + // LoopbackDSR specifies whether to support loopback direct server return. + LoopbackDSR bool `json:"loopbackDSR,omitempty"` } type RuntimeDNS struct { @@ -54,42 +55,67 @@ type PortMapEntry struct { HostIP string `json:"hostIP,omitempty"` } +// constants of the supported Windows Socket protocol, +// ref to https://docs.microsoft.com/en-us/dotnet/api/system.net.sockets.protocoltype. +var protocolEnums = map[string]uint32{ + "icmpv4": 1, + "igmp": 2, + "tcp": 6, + "udp": 17, + "icmpv6": 58, +} + +func (p *PortMapEntry) GetProtocolEnum() (uint32, error) { + var u, err = strconv.ParseUint(p.Protocol, 0, 10) + if err != nil { + var pe, exist = protocolEnums[strings.ToLower(p.Protocol)] + if !exist { + return 0, errors.New("invalid protocol supplied to port mapping policy") + } + return pe, nil + } + return uint32(u), nil +} + type RuntimeConfig struct { DNS RuntimeDNS `json:"dns"` PortMaps []PortMapEntry `json:"portMappings,omitempty"` } -type policy struct { +type Policy struct { Name string `json:"name"` Value json.RawMessage `json:"value"` } -func GetDefaultDestinationPrefix(ip *net.IP) string { - destinationPrefix := "0.0.0.0/0" - if ipv6 := ip.To4(); ipv6 == nil { - destinationPrefix = "::/0" +// GetHNSEndpointPolicies converts the configuration policies to HNSEndpoint policies. +func (n *NetConf) GetHNSEndpointPolicies() []json.RawMessage { + result := make([]json.RawMessage, 0, len(n.Policies)) + for _, p := range n.Policies { + if !strings.EqualFold(p.Name, "EndpointPolicy") { + continue + } + result = append(result, p.Value) } - return destinationPrefix + return result } -func (n *NetConf) ApplyLoopbackDSR(ip *net.IP) { - value := fmt.Sprintf(`"Destinations" : ["%s"]`, ip.String()) - if n.ApiVersion == 2 { - hcnLoopbackRoute := hcn.EndpointPolicy{ - Type: "OutBoundNAT", - Settings: []byte(fmt.Sprintf("{%s}", value)), +// GetHostComputeEndpointPolicies converts the configuration policies to HostComputeEndpoint policies. +func (n *NetConf) GetHostComputeEndpointPolicies() []hcn.EndpointPolicy { + result := make([]hcn.EndpointPolicy, 0, len(n.Policies)) + for _, p := range n.Policies { + if !strings.EqualFold(p.Name, "EndpointPolicy") { + continue } - n.HcnPolicyArgs = append(n.HcnPolicyArgs, hcnLoopbackRoute) - } else { - hnsLoopbackRoute := policy{ - Name: "EndpointPolicy", - Value: []byte(fmt.Sprintf(`{"Type": "OutBoundNAT", %s}`, value)), + var policy hcn.EndpointPolicy + if err := json.Unmarshal(p.Value, &policy); err != nil { + continue } - n.Policies = append(n.Policies, hnsLoopbackRoute) + result = append(result, policy) } + return result } -// If runtime dns values are there use that else use cni conf supplied dns +// GetDNS returns the DNS values if they are there use that else use netconf supplied DNS. func (n *NetConf) GetDNS() types.DNS { dnsResult := n.DNS if len(n.RuntimeConfig.DNS.Nameservers) > 0 { @@ -101,136 +127,222 @@ func (n *NetConf) GetDNS() types.DNS { return dnsResult } -// MarshalPolicies converts the Endpoint policies in Policies -// to HNS specific policies as Json raw bytes -func (n *NetConf) MarshalPolicies() []json.RawMessage { - if n.Policies == nil { - n.Policies = make([]policy, 0) +// ApplyLoopbackDSRPolicy configures the given IP to support loopback DSR. +func (n *NetConf) ApplyLoopbackDSRPolicy(ip *net.IP) { + if err := hcn.DSRSupported(); err != nil || ip == nil { + return } - result := make([]json.RawMessage, 0, len(n.Policies)) - for _, p := range n.Policies { + toPolicyValue := func(addr string) json.RawMessage { + if n.ApiVersion == 2 { + return bprintf(`{"Type": "OutBoundNAT", "Settings": {"Destinations": ["%s"]}}`, addr) + } + return bprintf(`{"Type": "OutBoundNAT", "Destinations": ["%s"]}`, addr) + } + ipBytes := []byte(ip.String()) + + // find OutBoundNAT policy + for i := range n.Policies { + p := &n.Policies[i] if !strings.EqualFold(p.Name, "EndpointPolicy") { continue } - result = append(result, p.Value) - } - - return result -} - -// ApplyOutboundNatPolicy applies NAT Policy in VFP using HNS -// Simultaneously an exception is added for the network that has to be Nat'd -func (n *NetConf) ApplyOutboundNatPolicy(nwToNat string) { - if n.Policies == nil { - n.Policies = make([]policy, 0) - } - - nwToNatBytes := []byte(nwToNat) - - for i, p := range n.Policies { - if !strings.EqualFold(p.Name, "EndpointPolicy") { + // filter OutBoundNAT policy + typeValue, _ := jsonparser.GetUnsafeString(p.Value, "Type") + if typeValue != "OutBoundNAT" { continue } - typeValue, err := jsonparser.GetUnsafeString(p.Value, "Type") - if err != nil || len(typeValue) == 0 { + // parse destination address list + var ( + destinationsValue []byte + dt jsonparser.ValueType + ) + if n.ApiVersion == 2 { + destinationsValue, dt, _, _ = jsonparser.Get(p.Value, "Settings", "Destinations") + } else { + destinationsValue, dt, _, _ = jsonparser.Get(p.Value, "Destinations") + } + + // skip if Destinations/DestinationList field is not found + if dt == jsonparser.NotExist { continue } - if !strings.EqualFold(typeValue, "OutBoundNAT") { - continue - } - - exceptionListValue, dt, _, _ := jsonparser.Get(p.Value, "ExceptionList") - // OutBoundNAT must with ExceptionList, so don't need to judge jsonparser.NotExist + // return if found the given address if dt == jsonparser.Array { - buf := bytes.Buffer{} - buf.WriteString(`{"Type": "OutBoundNAT", "ExceptionList": [`) - - jsonparser.ArrayEach(exceptionListValue, func(value []byte, dataType jsonparser.ValueType, offset int, err error) { + var found bool + _, _ = jsonparser.ArrayEach(destinationsValue, func(value []byte, dataType jsonparser.ValueType, offset int, err error) { if dataType == jsonparser.String && len(value) != 0 { - if bytes.Compare(value, nwToNatBytes) != 0 { - buf.WriteByte('"') - buf.Write(value) - buf.WriteByte('"') - buf.WriteByte(',') + if bytes.Compare(value, ipBytes) == 0 { + found = true } } }) - - buf.WriteString(`"` + nwToNat + `"]}`) - - n.Policies[i] = policy{ - Name: "EndpointPolicy", - Value: buf.Bytes(), - } - } else { - n.Policies[i] = policy{ - Name: "EndpointPolicy", - Value: []byte(`{"Type": "OutBoundNAT", "ExceptionList": ["` + nwToNat + `"]}`), + if found { + return } } - - return } - // didn't find the policyArg, add it - n.Policies = append(n.Policies, policy{ + // or add a new OutBoundNAT if not found + n.Policies = append(n.Policies, Policy{ Name: "EndpointPolicy", - Value: []byte(`{"Type": "OutBoundNAT", "ExceptionList": ["` + nwToNat + `"]}`), + Value: toPolicyValue(ip.String()), }) } -// ApplyDefaultPAPolicy is used to configure a endpoint PA policy in HNS -func (n *NetConf) ApplyDefaultPAPolicy(paAddress string) { - if n.Policies == nil { - n.Policies = make([]policy, 0) +// ApplyOutboundNatPolicy applies the sNAT policy in HNS/HCN and configures the given CIDR as an exception. +func (n *NetConf) ApplyOutboundNatPolicy(exceptionCIDR string) { + if exceptionCIDR == "" { + return } - // if its already present, leave untouched - for i, p := range n.Policies { + toPolicyValue := func(cidr ...string) json.RawMessage { + if n.ApiVersion == 2 { + return bprintf(`{"Type": "OutBoundNAT", "Settings": {"Exceptions": ["%s"]}}`, strings.Join(cidr, `","`)) + } + return bprintf(`{"Type": "OutBoundNAT", "ExceptionList": ["%s"]}`, strings.Join(cidr, `","`)) + } + exceptionCIDRBytes := []byte(exceptionCIDR) + + // find OutBoundNAT policy + for i := range n.Policies { + p := &n.Policies[i] if !strings.EqualFold(p.Name, "EndpointPolicy") { continue } - paValue, dt, _, _ := jsonparser.Get(p.Value, "PA") + // filter OutBoundNAT policy + typeValue, _ := jsonparser.GetUnsafeString(p.Value, "Type") + if typeValue != "OutBoundNAT" { + continue + } + + // parse exception CIDR list + var ( + exceptionsValue []byte + dt jsonparser.ValueType + ) + if n.ApiVersion == 2 { + exceptionsValue, dt, _, _ = jsonparser.Get(p.Value, "Settings", "Exceptions") + } else { + exceptionsValue, dt, _, _ = jsonparser.Get(p.Value, "ExceptionList") + } + + // skip if Exceptions/ExceptionList field is not found if dt == jsonparser.NotExist { continue - } else if dt == jsonparser.String && len(paValue) != 0 { - // found it, don't override - return } - n.Policies[i] = policy{ - Name: "EndpointPolicy", - Value: []byte(`{"Type": "PA", "PA": "` + paAddress + `"}`), + // return if found the given CIDR + if dt == jsonparser.Array { + var found bool + _, _ = jsonparser.ArrayEach(exceptionsValue, func(value []byte, dataType jsonparser.ValueType, offset int, err error) { + if dataType == jsonparser.String && len(value) != 0 { + if bytes.Compare(value, exceptionCIDRBytes) == 0 { + found = true + } + } + }) + if found { + return + } } - return } - // didn't find the policyArg, add it - n.Policies = append(n.Policies, policy{ + // or add a new OutBoundNAT if not found + n.Policies = append(n.Policies, Policy{ Name: "EndpointPolicy", - Value: []byte(`{"Type": "PA", "PA": "` + paAddress + `"}`), + Value: toPolicyValue(exceptionCIDR), }) } -// ApplyPortMappingPolicy is used to configure HostPort<>ContainerPort mapping in HNS -func (n *NetConf) ApplyPortMappingPolicy(portMappings []PortMapEntry) { - if portMappings == nil { +// ApplyDefaultPAPolicy applies an endpoint PA policy in HNS/HCN. +func (n *NetConf) ApplyDefaultPAPolicy(address string) { + if address == "" { return } - if n.Policies == nil { - n.Policies = make([]policy, 0) + toPolicyValue := func(addr string) json.RawMessage { + if n.ApiVersion == 2 { + return bprintf(`{"Type": "ProviderAddress", "Settings": {"ProviderAddress": "%s"}}`, addr) + } + return bprintf(`{"Type": "PA", "PA": "%s"}`, addr) + } + addressBytes := []byte(address) + + // find ProviderAddress policy + for i := range n.Policies { + p := &n.Policies[i] + if !strings.EqualFold(p.Name, "EndpointPolicy") { + continue + } + + // filter ProviderAddress policy + typeValue, _ := jsonparser.GetUnsafeString(p.Value, "Type") + if typeValue != "PA" && typeValue != "ProviderAddress" { + continue + } + + // parse provider address + var ( + paValue []byte + dt jsonparser.ValueType + ) + if n.ApiVersion == 2 { + paValue, dt, _, _ = jsonparser.Get(p.Value, "Settings", "ProviderAddress") + } else { + paValue, dt, _, _ = jsonparser.Get(p.Value, "PA") + } + + // skip if ProviderAddress/PA field is not found + if dt == jsonparser.NotExist { + continue + } + + // return if found the given address + if dt == jsonparser.String && bytes.Compare(paValue, addressBytes) == 0 { + return + } } - for _, portMapping := range portMappings { - n.Policies = append(n.Policies, policy{ + // or add a new ProviderAddress if not found + n.Policies = append(n.Policies, Policy{ + Name: "EndpointPolicy", + Value: toPolicyValue(address), + }) +} + +// ApplyPortMappingPolicy applies the host/container port mapping policies in HNS/HCN. +func (n *NetConf) ApplyPortMappingPolicy(portMappings []PortMapEntry) { + if len(portMappings) == 0 { + return + } + + toPolicyValue := func(p *PortMapEntry) json.RawMessage { + if n.ApiVersion == 2 { + var protocolEnum, _ = p.GetProtocolEnum() + return bprintf(`{"Type": "PortMapping", "Settings": {"InternalPort": %d, "ExternalPort": %d, "Protocol": %d, "VIP": "%s"}}`, p.ContainerPort, p.HostPort, protocolEnum, p.HostIP) + } + return bprintf(`{"Type": "NAT", "InternalPort": %d, "ExternalPort": %d, "Protocol": "%s"}`, p.ContainerPort, p.HostPort, p.Protocol) + } + + for i := range portMappings { + p := &portMappings[i] + // skip the invalid protocol mapping + if _, err := p.GetProtocolEnum(); err != nil { + continue + } + n.Policies = append(n.Policies, Policy{ Name: "EndpointPolicy", - Value: []byte(fmt.Sprintf(`{"Type": "NAT", "InternalPort": %d, "ExternalPort": %d, "Protocol": "%s"}`, portMapping.ContainerPort, portMapping.HostPort, portMapping.Protocol)), + Value: toPolicyValue(p), }) } } + +// bprintf is similar to fmt.Sprintf and returns a byte array as result. +func bprintf(format string, a ...interface{}) []byte { + return []byte(fmt.Sprintf(format, a...)) +} diff --git a/pkg/hns/netconf_windows_test.go b/pkg/hns/netconf_windows_test.go index 4bfa18af..3d2d9a66 100644 --- a/pkg/hns/netconf_windows_test.go +++ b/pkg/hns/netconf_windows_test.go @@ -15,221 +15,585 @@ package hns import ( "encoding/json" + "net" + "github.com/Microsoft/hcsshim/hcn" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" ) -var _ = Describe("HNS NetConf", func() { - Describe("ApplyOutBoundNATPolicy", func() { - Context("when not set by user", func() { - It("sets it by adding a policy", func() { +var _ = Describe("NetConf", func() { + Describe("ApplyLoopbackDSRPolicy", func() { + Context("via v1 api", func() { + var n NetConf + BeforeEach(func() { + n = NetConf{} + }) - // apply it - n := NetConf{} - n.ApplyOutboundNatPolicy("192.168.0.0/16") + It("filter out duplicated IP", func() { + // mock duplicated IP + ip := net.ParseIP("172.16.0.12") + n.ApplyLoopbackDSRPolicy(&ip) + n.ApplyLoopbackDSRPolicy(&ip) + // only one policy addlArgs := n.Policies Expect(addlArgs).Should(HaveLen(1)) + // normal type judgement policy := addlArgs[0] Expect(policy.Name).Should(Equal("EndpointPolicy")) - value := make(map[string]interface{}) json.Unmarshal(policy.Value, &value) - Expect(value).Should(HaveKey("Type")) - Expect(value).Should(HaveKey("ExceptionList")) Expect(value["Type"]).Should(Equal("OutBoundNAT")) + Expect(value).Should(HaveKey("Destinations")) - exceptionList := value["ExceptionList"].([]interface{}) - Expect(exceptionList).Should(HaveLen(1)) - Expect(exceptionList[0].(string)).Should(Equal("192.168.0.0/16")) + // and only one item + destinationList := value["Destinations"].([]interface{}) + Expect(destinationList).Should(HaveLen(1)) + Expect(destinationList[0].(string)).Should(Equal("172.16.0.12")) + }) + + It("append different IP", func() { + // mock different IP + ip1 := net.ParseIP("172.16.0.12") + n.ApplyLoopbackDSRPolicy(&ip1) + ip2 := net.ParseIP("172.16.0.13") + n.ApplyLoopbackDSRPolicy(&ip2) + + // will be two policies + addlArgs := n.Policies + Expect(addlArgs).Should(HaveLen(2)) + + // normal type judgement + policy := addlArgs[1] // pick second item + Expect(policy.Name).Should(Equal("EndpointPolicy")) + value := make(map[string]interface{}) + json.Unmarshal(policy.Value, &value) + Expect(value).Should(HaveKey("Type")) + Expect(value["Type"]).Should(Equal("OutBoundNAT")) + Expect(value).Should(HaveKey("Destinations")) + + // only one item + destinationList := value["Destinations"].([]interface{}) + Expect(destinationList).Should(HaveLen(1)) + Expect(destinationList[0].(string)).Should(Equal("172.16.0.13")) }) }) - Context("when set by user", func() { - It("appends exceptions to the existing policy", func() { - // first set it - n := NetConf{} - n.ApplyOutboundNatPolicy("192.168.0.0/16") + Context("via v2 api", func() { + var n NetConf + BeforeEach(func() { + n = NetConf{ApiVersion: 2} + }) - // then attempt to update it - n.ApplyOutboundNatPolicy("10.244.0.0/16") + It("filter out duplicated IP", func() { + // mock duplicated IP + ip := net.ParseIP("172.16.0.12") + n.ApplyLoopbackDSRPolicy(&ip) + n.ApplyLoopbackDSRPolicy(&ip) - // it should be unchanged! + // only one policy addlArgs := n.Policies Expect(addlArgs).Should(HaveLen(1)) + // normal type judgement policy := addlArgs[0] Expect(policy.Name).Should(Equal("EndpointPolicy")) - - var value map[string]interface{} + value := make(map[string]interface{}) json.Unmarshal(policy.Value, &value) - Expect(value).Should(HaveKey("Type")) - Expect(value).Should(HaveKey("ExceptionList")) Expect(value["Type"]).Should(Equal("OutBoundNAT")) + Expect(value).Should(HaveKey("Settings")) + // and only one item + settings := value["Settings"].(map[string]interface{}) + destinationList := settings["Destinations"].([]interface{}) + Expect(destinationList).Should(HaveLen(1)) + Expect(destinationList[0].(string)).Should(Equal("172.16.0.12")) + }) + + It("append different IP", func() { + // mock different IP + ip1 := net.ParseIP("172.16.0.12") + n.ApplyLoopbackDSRPolicy(&ip1) + ip2 := net.ParseIP("172.16.0.13") + n.ApplyLoopbackDSRPolicy(&ip2) + + // will be two policies + addlArgs := n.Policies + Expect(addlArgs).Should(HaveLen(2)) + + // normal type judgement + policy := addlArgs[1] // pick second item + Expect(policy.Name).Should(Equal("EndpointPolicy")) + value := make(map[string]interface{}) + json.Unmarshal(policy.Value, &value) + Expect(value).Should(HaveKey("Type")) + Expect(value["Type"]).Should(Equal("OutBoundNAT")) + Expect(value).Should(HaveKey("Settings")) + + // only one item + settings := value["Settings"].(map[string]interface{}) + destinationList := settings["Destinations"].([]interface{}) + Expect(destinationList).Should(HaveLen(1)) + Expect(destinationList[0].(string)).Should(Equal("172.16.0.13")) + }) + }) + }) + + Describe("ApplyOutBoundNATPolicy", func() { + Context("via v1 api", func() { + var n NetConf + BeforeEach(func() { + n = NetConf{} + }) + + It("append different IP", func() { + // mock different IP + n.ApplyOutboundNatPolicy("192.168.0.0/16") + n.ApplyOutboundNatPolicy("10.244.0.0/16") + + // will be two policies + addlArgs := n.Policies + Expect(addlArgs).Should(HaveLen(2)) + + // normal type judgement + policy := addlArgs[1] // pick second item + Expect(policy.Name).Should(Equal("EndpointPolicy")) + value := make(map[string]interface{}) + json.Unmarshal(policy.Value, &value) + Expect(value).Should(HaveKey("Type")) + Expect(value["Type"]).Should(Equal("OutBoundNAT")) + Expect(value).Should(HaveKey("ExceptionList")) + + // but get two items exceptionList := value["ExceptionList"].([]interface{}) - Expect(exceptionList).Should(HaveLen(2)) - Expect(exceptionList[0].(string)).Should(Equal("192.168.0.0/16")) - Expect(exceptionList[1].(string)).Should(Equal("10.244.0.0/16")) + Expect(exceptionList).Should(HaveLen(1)) + Expect(exceptionList[0].(string)).Should(Equal("10.244.0.0/16")) + }) + + It("append a new one if there is not an exception OutBoundNAT policy", func() { + // mock different OutBoundNAT routes + n.Policies = []Policy{ + { + Name: "EndpointPolicy", + Value: bprintf(`{"Type": "OutBoundNAT", "OtherList": []}`), + }, + } + n.ApplyOutboundNatPolicy("10.244.0.0/16") + + // has two policies + addlArgs := n.Policies + Expect(addlArgs).Should(HaveLen(2)) + + // normal type judgement + policy := addlArgs[0] + Expect(policy.Name).Should(Equal("EndpointPolicy")) + value := make(map[string]interface{}) + json.Unmarshal(policy.Value, &value) + Expect(value).Should(HaveKey("Type")) + Expect(value["Type"]).Should(Equal("OutBoundNAT")) + Expect(value).Should(HaveKey("OtherList")) + policy = addlArgs[1] + value = make(map[string]interface{}) + json.Unmarshal(policy.Value, &value) + Expect(value).Should(HaveKey("Type")) + Expect(value["Type"]).Should(Equal("OutBoundNAT")) + Expect(value).Should(HaveKey("ExceptionList")) + + // only get one item + exceptionList := value["ExceptionList"].([]interface{}) + Expect(exceptionList).Should(HaveLen(1)) + Expect(exceptionList[0].(string)).Should(Equal("10.244.0.0/16")) + }) + + It("nothing to do if CIDR is blank", func() { + // mock different OutBoundNAT routes + n.Policies = []Policy{ + { + Name: "EndpointPolicy", + Value: bprintf(`{"Type": "OutBoundNAT", "ExceptionList": []}`), + }, + } + n.ApplyOutboundNatPolicy("") + + // only one policy + addlArgs := n.Policies + Expect(addlArgs).Should(HaveLen(1)) + + // normal type judgement + policy := addlArgs[0] + Expect(policy.Name).Should(Equal("EndpointPolicy")) + value := make(map[string]interface{}) + json.Unmarshal(policy.Value, &value) + Expect(value).Should(HaveKey("Type")) + Expect(value["Type"]).Should(Equal("OutBoundNAT")) + Expect(value).Should(HaveKey("ExceptionList")) + + // empty list + Expect(value["ExceptionList"]).ShouldNot(BeNil()) + Expect(value["ExceptionList"]).Should(HaveLen(0)) + }) + }) + + Context("via v2 api", func() { + var n NetConf + BeforeEach(func() { + n = NetConf{ApiVersion: 2} + }) + + It("append different IP", func() { + // mock different IP + n.ApplyOutboundNatPolicy("192.168.0.0/16") + n.ApplyOutboundNatPolicy("10.244.0.0/16") + + // will be two policies + addlArgs := n.Policies + Expect(addlArgs).Should(HaveLen(2)) + + // normal type judgement + policy := addlArgs[1] // pick second item + Expect(policy.Name).Should(Equal("EndpointPolicy")) + value := make(map[string]interface{}) + json.Unmarshal(policy.Value, &value) + Expect(value).Should(HaveKey("Type")) + Expect(value["Type"]).Should(Equal("OutBoundNAT")) + Expect(value).Should(HaveKey("Settings")) + + // but get two items + settings := value["Settings"].(map[string]interface{}) + exceptionList := settings["Exceptions"].([]interface{}) + Expect(exceptionList).Should(HaveLen(1)) + Expect(exceptionList[0].(string)).Should(Equal("10.244.0.0/16")) + }) + + It("append a new one if there is not an exception OutBoundNAT policy", func() { + // mock different OutBoundNAT routes + n.Policies = []Policy{ + { + Name: "EndpointPolicy", + Value: bprintf(`{"Type": "OutBoundNAT", "Settings": {"Others": []}}`), + }, + } + n.ApplyOutboundNatPolicy("10.244.0.0/16") + + // has two policies + addlArgs := n.Policies + Expect(addlArgs).Should(HaveLen(2)) + + // normal type judgement + policy := addlArgs[0] + Expect(policy.Name).Should(Equal("EndpointPolicy")) + value := make(map[string]interface{}) + json.Unmarshal(policy.Value, &value) + Expect(value).Should(HaveKey("Type")) + Expect(value["Type"]).Should(Equal("OutBoundNAT")) + Expect(value).Should(HaveKey("Settings")) + Expect(value["Settings"]).Should(HaveKey("Others")) + policy = addlArgs[1] + value = make(map[string]interface{}) + json.Unmarshal(policy.Value, &value) + Expect(value).Should(HaveKey("Type")) + Expect(value["Type"]).Should(Equal("OutBoundNAT")) + Expect(value).Should(HaveKey("Settings")) + + // only get one item + settings := value["Settings"].(map[string]interface{}) + exceptionList := settings["Exceptions"].([]interface{}) + Expect(exceptionList).Should(HaveLen(1)) + Expect(exceptionList[0].(string)).Should(Equal("10.244.0.0/16")) + }) + + It("nothing to do if CIDR is blank", func() { + // mock different OutBoundNAT routes + n.Policies = []Policy{ + { + Name: "EndpointPolicy", + Value: bprintf(`{"Type": "OutBoundNAT", "Settings": {"Exceptions": []}}`), + }, + } + n.ApplyOutboundNatPolicy("") + + // only one policy + addlArgs := n.Policies + Expect(addlArgs).Should(HaveLen(1)) + + // normal type judgement + policy := addlArgs[0] + Expect(policy.Name).Should(Equal("EndpointPolicy")) + value := make(map[string]interface{}) + json.Unmarshal(policy.Value, &value) + Expect(value).Should(HaveKey("Type")) + Expect(value["Type"]).Should(Equal("OutBoundNAT")) + Expect(value).Should(HaveKey("Settings")) + + // empty list + settings := value["Settings"].(map[string]interface{}) + Expect(settings["Exceptions"]).ShouldNot(BeNil()) + Expect(settings["Exceptions"]).Should(HaveLen(0)) }) }) }) Describe("ApplyDefaultPAPolicy", func() { - Context("when not set by user", func() { - It("sets it by adding a policy", func() { - - n := NetConf{} - n.ApplyDefaultPAPolicy("192.168.0.1") - - addlArgs := n.Policies - Expect(addlArgs).Should(HaveLen(1)) - - policy := addlArgs[0] - Expect(policy.Name).Should(Equal("EndpointPolicy")) - - value := make(map[string]interface{}) - json.Unmarshal(policy.Value, &value) - - Expect(value).Should(HaveKey("Type")) - Expect(value["Type"]).Should(Equal("PA")) - - paAddress := value["PA"].(string) - Expect(paAddress).Should(Equal("192.168.0.1")) + Context("via v1 api", func() { + var n NetConf + BeforeEach(func() { + n = NetConf{} }) - }) - Context("when set by user", func() { - It("does not override", func() { - n := NetConf{} + It("append different IP", func() { + // mock different IP n.ApplyDefaultPAPolicy("192.168.0.1") n.ApplyDefaultPAPolicy("192.168.0.2") + // will be two policies addlArgs := n.Policies - Expect(addlArgs).Should(HaveLen(1)) + Expect(addlArgs).Should(HaveLen(2)) - policy := addlArgs[0] + // normal type judgement + policy := addlArgs[1] // judge second item Expect(policy.Name).Should(Equal("EndpointPolicy")) - value := make(map[string]interface{}) json.Unmarshal(policy.Value, &value) - Expect(value).Should(HaveKey("Type")) Expect(value["Type"]).Should(Equal("PA")) + // compare with second item paAddress := value["PA"].(string) - Expect(paAddress).Should(Equal("192.168.0.1")) - Expect(paAddress).ShouldNot(Equal("192.168.0.2")) + Expect(paAddress).Should(Equal("192.168.0.2")) + }) + + It("nothing to do if IP is blank", func() { + // mock different policy + n.Policies = []Policy{ + { + Name: "EndpointPolicy", + Value: bprintf(`{"Type": "OutBoundNAT", "Exceptions": ["192.168.0.0/16"]}`), + }, + } + n.ApplyDefaultPAPolicy("") + + // nothing + addlArgs := n.Policies + Expect(addlArgs).Should(HaveLen(1)) + }) + }) + + Context("via v2 api", func() { + var n NetConf + BeforeEach(func() { + n = NetConf{ApiVersion: 2} + }) + + It("append different IP", func() { + // mock different IP + n.ApplyDefaultPAPolicy("192.168.0.1") + n.ApplyDefaultPAPolicy("192.168.0.2") + + // will be two policies + addlArgs := n.Policies + Expect(addlArgs).Should(HaveLen(2)) + + // normal type judgement + policy := addlArgs[1] // judge second item + Expect(policy.Name).Should(Equal("EndpointPolicy")) + value := make(map[string]interface{}) + json.Unmarshal(policy.Value, &value) + Expect(value).Should(HaveKey("Type")) + Expect(value["Type"]).Should(Equal("ProviderAddress")) + Expect(value).Should(HaveKey("Settings")) + + // compare with second item + settings := value["Settings"].(map[string]interface{}) + paAddress := settings["ProviderAddress"].(string) + Expect(paAddress).Should(Equal("192.168.0.2")) + }) + + It("nothing to do if IP is blank", func() { + // mock different policy + n.Policies = []Policy{ + { + Name: "EndpointPolicy", + Value: bprintf(`{"Type": "OutBoundNAT", "Settings": {"Exceptions": ["192.168.0.0/16"]}}`), + }, + } + n.ApplyDefaultPAPolicy("") + + // nothing + addlArgs := n.Policies + Expect(addlArgs).Should(HaveLen(1)) }) }) }) Describe("ApplyPortMappingPolicy", func() { - Context("when portMappings not activated", func() { - It("does nothing", func() { - n := NetConf{} + Context("via v1 api", func() { + var n NetConf + BeforeEach(func() { + n = NetConf{} + }) + + It("nothing to do if input is empty", func() { n.ApplyPortMappingPolicy(nil) Expect(n.Policies).Should(BeNil()) n.ApplyPortMappingPolicy([]PortMapEntry{}) - Expect(n.Policies).Should(HaveLen(0)) + Expect(n.Policies).Should(BeNil()) }) - }) - Context("when portMappings is activated", func() { - It("creates NAT policies", func() { - n := NetConf{} + It("create one NAT policy", func() { + // mock different IP n.ApplyPortMappingPolicy([]PortMapEntry{ { ContainerPort: 80, HostPort: 8080, Protocol: "TCP", - HostIP: "ignored", + HostIP: "192.168.1.2", }, }) - Expect(n.Policies).Should(HaveLen(1)) + // only one item + addlArgs := n.Policies + Expect(addlArgs).Should(HaveLen(1)) - policy := n.Policies[0] + // normal type judgement + policy := addlArgs[0] Expect(policy.Name).Should(Equal("EndpointPolicy")) - value := make(map[string]interface{}) json.Unmarshal(policy.Value, &value) - Expect(value).Should(HaveKey("Type")) Expect(value["Type"]).Should(Equal("NAT")) + // compare all values Expect(value).Should(HaveKey("InternalPort")) Expect(value["InternalPort"]).Should(Equal(float64(80))) - Expect(value).Should(HaveKey("ExternalPort")) Expect(value["ExternalPort"]).Should(Equal(float64(8080))) - Expect(value).Should(HaveKey("Protocol")) Expect(value["Protocol"]).Should(Equal("TCP")) }) }) - }) - Describe("MarshalPolicies", func() { - Context("when not set by user", func() { - It("sets it by adding a policy", func() { - - n := NetConf{ - Policies: []policy{ - { - Name: "EndpointPolicy", - Value: []byte(`{"someKey": "someValue"}`), - }, - { - Name: "someOtherType", - Value: []byte(`{"someOtherKey": "someOtherValue"}`), - }, - }, - } - - result := n.MarshalPolicies() - Expect(len(result)).To(Equal(1)) - - policy := make(map[string]interface{}) - err := json.Unmarshal(result[0], &policy) - Expect(err).ToNot(HaveOccurred()) - Expect(policy).Should(HaveKey("someKey")) - Expect(policy["someKey"]).To(Equal("someValue")) + Context("via v2 api", func() { + var n NetConf + BeforeEach(func() { + n = NetConf{ApiVersion: 2} }) - }) - Context("when set by user", func() { - It("appends exceptions to the existing policy", func() { - // first set it - n := NetConf{} - n.ApplyOutboundNatPolicy("192.168.0.0/16") + It("nothing to do if input is empty", func() { + n.ApplyPortMappingPolicy(nil) + Expect(n.Policies).Should(BeNil()) - // then attempt to update it - n.ApplyOutboundNatPolicy("10.244.0.0/16") + n.ApplyPortMappingPolicy([]PortMapEntry{}) + Expect(n.Policies).Should(BeNil()) + }) - // it should be unchanged! + It("creates one NAT policy", func() { + // mock different IP + n.ApplyPortMappingPolicy([]PortMapEntry{ + { + ContainerPort: 80, + HostPort: 8080, + Protocol: "TCP", + HostIP: "192.168.1.2", + }, + }) + + // only one item addlArgs := n.Policies Expect(addlArgs).Should(HaveLen(1)) + // normal type judgement policy := addlArgs[0] Expect(policy.Name).Should(Equal("EndpointPolicy")) - - var value map[string]interface{} + value := make(map[string]interface{}) json.Unmarshal(policy.Value, &value) - Expect(value).Should(HaveKey("Type")) - Expect(value).Should(HaveKey("ExceptionList")) - Expect(value["Type"]).Should(Equal("OutBoundNAT")) + Expect(value["Type"]).Should(Equal("PortMapping")) + Expect(value).Should(HaveKey("Settings")) - exceptionList := value["ExceptionList"].([]interface{}) - Expect(exceptionList).Should(HaveLen(2)) - Expect(exceptionList[0].(string)).Should(Equal("192.168.0.0/16")) - Expect(exceptionList[1].(string)).Should(Equal("10.244.0.0/16")) + // compare all values + settings := value["Settings"].(map[string]interface{}) + Expect(settings).Should(HaveKey("InternalPort")) + Expect(settings["InternalPort"]).Should(Equal(float64(80))) + Expect(settings).Should(HaveKey("ExternalPort")) + Expect(settings["ExternalPort"]).Should(Equal(float64(8080))) + Expect(settings).Should(HaveKey("Protocol")) + Expect(settings["Protocol"]).Should(Equal(float64(6))) + Expect(settings).Should(HaveKey("VIP")) + Expect(settings["VIP"]).Should(Equal("192.168.1.2")) + }) + }) + }) + + Describe("GetXEndpointPolicies", func() { + Context("via v1 api", func() { + var n NetConf + BeforeEach(func() { + n = NetConf{} + }) + + It("GetHNSEndpointPolicies", func() { + // mock different policies + n.Policies = []Policy{ + { + Name: "EndpointPolicy", + Value: []byte(`{"Type": "OutBoundNAT", "ExceptionList": [ "192.168.1.2" ]}`), + }, + { + Name: "someOtherType", + Value: []byte(`{"someOtherKey": "someOtherValue"}`), + }, + } + + // only one valid item + result := n.GetHNSEndpointPolicies() + Expect(len(result)).To(Equal(1)) + + // normal type judgement + policy := make(map[string]interface{}) + err := json.Unmarshal(result[0], &policy) + Expect(err).ToNot(HaveOccurred()) + Expect(policy).Should(HaveKey("Type")) + Expect(policy["Type"]).To(Equal("OutBoundNAT")) + Expect(policy).Should(HaveKey("ExceptionList")) + Expect(policy["ExceptionList"]).To(ContainElement("192.168.1.2")) + }) + }) + + Context("via v2 api", func() { + var n NetConf + BeforeEach(func() { + n = NetConf{ApiVersion: 2} + }) + + It("GetHostComputeEndpointPolicies", func() { + // mock different policies + n.Policies = []Policy{ + { + Name: "EndpointPolicy", + Value: []byte(`{"Type": "OutBoundNAT", "Settings": {"Exceptions": [ "192.168.1.2" ]}}`), + }, + { + Name: "someOtherType", + Value: []byte(`{"someOtherKey": "someOtherValue"}`), + }, + } + + // only one valid item + result := n.GetHostComputeEndpointPolicies() + Expect(len(result)).To(Equal(1)) + + // normal type judgement + policy := result[0] + Expect(policy.Type).Should(Equal(hcn.OutBoundNAT)) + settings := make(map[string]interface{}) + err := json.Unmarshal(policy.Settings, &settings) + Expect(err).ToNot(HaveOccurred()) + Expect(settings["Exceptions"]).To(ContainElement("192.168.1.2")) }) }) }) diff --git a/plugins/main/windows/win-bridge/sample.conf b/plugins/main/windows/win-bridge/sample-v1.conf similarity index 100% rename from plugins/main/windows/win-bridge/sample.conf rename to plugins/main/windows/win-bridge/sample-v1.conf diff --git a/plugins/main/windows/win-bridge/sample-v2.conf b/plugins/main/windows/win-bridge/sample-v2.conf new file mode 100644 index 00000000..3aabbda8 --- /dev/null +++ b/plugins/main/windows/win-bridge/sample-v2.conf @@ -0,0 +1,52 @@ +{ + "name":"cbr0", + "type":"flannel", + "delegate":{ + "apiVersion":2, + "type":"win-bridge", + "dns":{ + "nameservers":[ + "11.0.0.10" + ], + "search":[ + "svc.cluster.local" + ] + }, + "policies":[ + { + "name":"EndpointPolicy", + "value":{ + "Type":"OutBoundNAT", + "Settings":{ + "Exceptions":[ + "192.168.0.0/16", + "11.0.0.0/8", + "10.137.196.0/23" + ] + } + } + }, + { + "name":"EndpointPolicy", + "value":{ + "Type":"SDNRoute", + "Settings":{ + "DestinationPrefix":"11.0.0.0/8", + "NeedEncap":true + } + } + }, + { + "name":"EndpointPolicy", + "value":{ + "Type":"SDNRoute", + "Settings":{ + "DestinationPrefix":"10.137.198.27/32", + "NeedEncap":true + } + } + } + ], + "loopbackDSR":true + } +} \ No newline at end of file diff --git a/plugins/main/windows/win-bridge/win-bridge_windows.go b/plugins/main/windows/win-bridge/win-bridge_windows.go index 36f1035a..0e2645fc 100644 --- a/plugins/main/windows/win-bridge/win-bridge_windows.go +++ b/plugins/main/windows/win-bridge/win-bridge_windows.go @@ -55,21 +55,22 @@ func loadNetConf(bytes []byte) (*NetConf, string, error) { return n, n.CNIVersion, nil } -func ProcessEndpointArgs(args *skel.CmdArgs, n *NetConf) (*hns.EndpointInfo, error) { +func processEndpointArgs(args *skel.CmdArgs, n *NetConf) (*hns.EndpointInfo, error) { epInfo := new(hns.EndpointInfo) epInfo.NetworkName = n.Name epInfo.EndpointName = hns.ConstructEndpointName(args.ContainerID, args.Netns, epInfo.NetworkName) - // It's not necessary to have have an IPAM in windows as hns can provide IP/GW + + // it's not necessary to have have an IPAM in windows as HNS can provide IP/GW if n.IPAM.Type != "" { r, err := ipam.ExecAdd(n.IPAM.Type, args.StdinData) if err != nil { - return nil, errors.Annotatef(err, "error while ipam.ExecAdd") + return nil, errors.Annotatef(err, "error while executing IPAM addition") } - // Convert whatever the IPAM result was into the current Result type + // convert whatever the IPAM result was into the current result result, err := current.NewResultFromResult(r) if err != nil { - return nil, errors.Annotatef(err, "error while NewResultFromResult") + return nil, errors.Annotatef(err, "error while converting the result from IPAM addition") } else { if len(result.IPs) == 0 { return nil, fmt.Errorf("IPAM plugin return is missing IP config") @@ -81,12 +82,11 @@ func ProcessEndpointArgs(args *skel.CmdArgs, n *NetConf) (*hns.EndpointInfo, err epInfo.Gateway[len(epInfo.Gateway)-1] += 2 } } - // NAT based on the the configured cluster network - if len(n.IPMasqNetwork) != 0 { - n.ApplyOutboundNatPolicy(n.IPMasqNetwork) - } - // Add HostPort mapping if any present + // configure sNAT exception + n.ApplyOutboundNatPolicy(n.IPMasqNetwork) + + // add port mapping if any present n.ApplyPortMappingPolicy(n.RuntimeConfig.PortMaps) epInfo.DNS = n.GetDNS() @@ -98,40 +98,37 @@ func cmdHnsAdd(args *skel.CmdArgs, n *NetConf) (*current.Result, error) { networkName := n.Name hnsNetwork, err := hcsshim.GetHNSNetworkByName(networkName) if err != nil { - return nil, errors.Annotatef(err, "error while GETHNSNewtorkByName(%s)", networkName) + return nil, errors.Annotatef(err, "error while getting network %v", networkName) } - if hnsNetwork == nil { - return nil, fmt.Errorf("network %v not found", networkName) + return nil, fmt.Errorf("network %v is not found", networkName) } - if !strings.EqualFold(hnsNetwork.Type, "L2Bridge") && !strings.EqualFold(hnsNetwork.Type, "L2Tunnel") { - return nil, fmt.Errorf("network %v is of an unexpected type: %v", networkName, hnsNetwork.Type) + return nil, fmt.Errorf("network %v is of unexpected type: %v", networkName, hnsNetwork.Type) } epName := hns.ConstructEndpointName(args.ContainerID, args.Netns, n.Name) - hnsEndpoint, err := hns.ProvisionEndpoint(epName, hnsNetwork.Id, args.ContainerID, args.Netns, func() (*hcsshim.HNSEndpoint, error) { - epInfo, err := ProcessEndpointArgs(args, n) + hnsEndpoint, err := hns.AddHnsEndpoint(epName, hnsNetwork.Id, args.ContainerID, args.Netns, func() (*hcsshim.HNSEndpoint, error) { + epInfo, err := processEndpointArgs(args, n) if err != nil { - return nil, errors.Annotatef(err, "error while ProcessEndpointArgs") + return nil, errors.Annotate(err, "error while processing endpoint args") } epInfo.NetworkId = hnsNetwork.Id hnsEndpoint, err := hns.GenerateHnsEndpoint(epInfo, &n.NetConf) if err != nil { - return nil, errors.Annotatef(err, "error while GenerateHnsEndpoint") + return nil, errors.Annotate(err, "error while generating HNSEndpoint") } return hnsEndpoint, nil }) if err != nil { - return nil, errors.Annotatef(err, "error while ProvisionEndpoint(%v,%v,%v)", epName, hnsNetwork.Id, args.ContainerID) + return nil, errors.Annotate(err, "error while adding HNSEndpoint") } - result, err := hns.ConstructResult(hnsNetwork, hnsEndpoint) + result, err := hns.ConstructHnsResult(hnsNetwork, hnsEndpoint) if err != nil { - return nil, errors.Annotatef(err, "error while constructResult") + return nil, errors.Annotate(err, "error while constructing HNSEndpoint addition result") } - return result, nil } @@ -139,48 +136,44 @@ func cmdHcnAdd(args *skel.CmdArgs, n *NetConf) (*current.Result, error) { networkName := n.Name hcnNetwork, err := hcn.GetNetworkByName(networkName) if err != nil { - return nil, errors.Annotatef(err, "error while GetNetworkByName(%s)", networkName) + return nil, errors.Annotatef(err, "error while getting network %v", networkName) } - if hcnNetwork == nil { - return nil, fmt.Errorf("network %v not found", networkName) + return nil, fmt.Errorf("network %v is not found", networkName) } - if hcnNetwork.Type != hcn.L2Bridge && hcnNetwork.Type != hcn.L2Tunnel { return nil, fmt.Errorf("network %v is of unexpected type: %v", networkName, hcnNetwork.Type) } epName := hns.ConstructEndpointName(args.ContainerID, args.Netns, n.Name) - hcnEndpoint, err := hns.AddHcnEndpoint(epName, hcnNetwork.Id, args.Netns, func() (*hcn.HostComputeEndpoint, error) { - epInfo, err := ProcessEndpointArgs(args, n) + epInfo, err := processEndpointArgs(args, n) if err != nil { - return nil, errors.Annotatef(err, "error while ProcessEndpointArgs") + return nil, errors.Annotate(err, "error while processing endpoint args") } epInfo.NetworkId = hcnNetwork.Id hcnEndpoint, err := hns.GenerateHcnEndpoint(epInfo, &n.NetConf) if err != nil { - return nil, errors.Annotatef(err, "error while GenerateHcnEndpoint") + return nil, errors.Annotate(err, "error while generating HostComputeEndpoint") } return hcnEndpoint, nil }) if err != nil { - return nil, errors.Annotatef(err, "error while AddHcnEndpoint(%v,%v,%v)", epName, hcnNetwork.Id, args.Netns) + return nil, errors.Annotate(err, "error while adding HostComputeEndpoint") } result, err := hns.ConstructHcnResult(hcnNetwork, hcnEndpoint) if err != nil { - return nil, errors.Annotatef(err, "error while ConstructHcnResult") + return nil, errors.Annotate(err, "error while constructing HostComputeEndpoint addition result") } - return result, nil } func cmdAdd(args *skel.CmdArgs) error { n, cniVersion, err := loadNetConf(args.StdinData) if err != nil { - return errors.Annotate(err, "error while loadNetConf") + return err } var result *current.Result @@ -189,15 +182,11 @@ func cmdAdd(args *skel.CmdArgs) error { } else { result, err = cmdHnsAdd(args, n) } - if err != nil { ipam.ExecDel(n.IPAM.Type, args.StdinData) - return errors.Annotate(err, "error while executing ADD command") + return err } - if result == nil { - return fmt.Errorf("result for ADD not populated correctly") - } return types.PrintResult(result, cniVersion) } @@ -216,9 +205,8 @@ func cmdDel(args *skel.CmdArgs) error { if n.ApiVersion == 2 { return hns.RemoveHcnEndpoint(epName) - } else { - return hns.DeprovisionEndpoint(epName, args.Netns, args.ContainerID) } + return hns.RemoveHnsEndpoint(epName, args.Netns, args.ContainerID) } func cmdCheck(_ *skel.CmdArgs) error { diff --git a/plugins/main/windows/win-overlay/win-overlay_windows.go b/plugins/main/windows/win-overlay/win-overlay_windows.go index 77bb1714..3882b9af 100644 --- a/plugins/main/windows/win-overlay/win-overlay_windows.go +++ b/plugins/main/windows/win-overlay/win-overlay_windows.go @@ -86,7 +86,7 @@ func cmdAdd(args *skel.CmdArgs) error { epName := hns.ConstructEndpointName(args.ContainerID, args.Netns, n.Name) - hnsEndpoint, err := hns.ProvisionEndpoint(epName, hnsNetwork.Id, args.ContainerID, args.Netns, func() (*hcsshim.HNSEndpoint, error) { + hnsEndpoint, err := hns.AddHnsEndpoint(epName, hnsNetwork.Id, args.ContainerID, args.Netns, func() (*hcsshim.HNSEndpoint, error) { // run the IPAM plugin and get back the config to apply r, err := ipam.ExecAdd(n.IPAM.Type, args.StdinData) if err != nil { @@ -119,7 +119,7 @@ func cmdAdd(args *skel.CmdArgs) error { result.DNS = n.GetDNS() if n.LoopbackDSR { - n.ApplyLoopbackDSR(&ipAddr) + n.ApplyLoopbackDSRPolicy(&ipAddr) } hnsEndpoint := &hcsshim.HNSEndpoint{ Name: epName, @@ -129,7 +129,7 @@ func cmdAdd(args *skel.CmdArgs) error { GatewayAddress: gw, IPAddress: ipAddr, MacAddress: macAddr, - Policies: n.MarshalPolicies(), + Policies: n.GetHNSEndpointPolicies(), } return hnsEndpoint, nil @@ -140,10 +140,10 @@ func cmdAdd(args *skel.CmdArgs) error { } }() if err != nil { - return errors.Annotatef(err, "error while ProvisionEndpoint(%v,%v,%v)", epName, hnsNetwork.Id, args.ContainerID) + return errors.Annotatef(err, "error while AddHnsEndpoint(%v,%v,%v)", epName, hnsNetwork.Id, args.ContainerID) } - result, err := hns.ConstructResult(hnsNetwork, hnsEndpoint) + result, err := hns.ConstructHnsResult(hnsNetwork, hnsEndpoint) if err != nil { return errors.Annotatef(err, "error while constructResult") } @@ -164,7 +164,7 @@ func cmdDel(args *skel.CmdArgs) error { epName := hns.ConstructEndpointName(args.ContainerID, args.Netns, n.Name) - return hns.DeprovisionEndpoint(epName, args.Netns, args.ContainerID) + return hns.RemoveHnsEndpoint(epName, args.Netns, args.ContainerID) } func cmdCheck(_ *skel.CmdArgs) error {