diff --git a/.appveyor.yml b/.appveyor.yml index ea06455d..1133a06e 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -6,15 +6,16 @@ environment: install: - echo %PATH% - echo %GOPATH% - - set PATH=%GOPATH%\bin;c:\go\bin;%PATH% - go version - go env + - ps: $webClient = New-Object System.Net.WebClient; $InstallPath="c:" ; $webClient.DownloadFile("https://raw.githubusercontent.com/jhowardmsft/docker-tdmgcc/master/gcc.zip", "$InstallPath\gcc.zip"); Expand-Archive $InstallPath\gcc.zip -DestinationPath $InstallPath\gcc -Force; $webClient.DownloadFile("https://raw.githubusercontent.com/jhowardmsft/docker-tdmgcc/master/runtime.zip", "$InstallPath\runtime.zip"); Expand-Archive $InstallPath\runtime.zip -DestinationPath $InstallPath\gcc -Force; $webClient.DownloadFile("https://raw.githubusercontent.com/jhowardmsft/docker-tdmgcc/master/binutils.zip","$InstallPath\binutils.zip"); Expand-Archive $InstallPath\binutils.zip -DestinationPath $InstallPath\gcc -Force; + - set PATH=%GOPATH%\bin;c:\go\bin;c:\gcc\bin;%PATH% build: off test_script: - ps: | - go list ./... | Select-String -Pattern (Get-Content "./plugins/linux_only.txt") -NotMatch > "to_test.txt" + go list ./... | Select-String -Pattern (Get-Content "./plugins/windows_only.txt") > "to_test.txt" echo "Will test:" Get-Content "to_test.txt" foreach ($pkg in Get-Content "to_test.txt") { diff --git a/.travis.yml b/.travis.yml index e6ab0e99..faa133e0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,6 +20,7 @@ matrix: fast_finish: true install: + - sudo apt-get install gcc-multilib gcc-mingw-w64 -y - go get github.com/onsi/ginkgo/ginkgo - go get github.com/containernetworking/cni/cnitool diff --git a/README.md b/README.md index 86460176..50644644 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,9 @@ Read [CONTRIBUTING](CONTRIBUTING.md) for build and test instructions. * `macvlan`: Creates a new MAC address, forwards all traffic to that to the container. * `ptp`: Creates a veth pair. * `vlan`: Allocates a vlan device. - +#### Windows: windows specific +* `win-bridge`: Creates a bridge, adds the host and the container to it. +* `win-overlay`: Creates an overlay interface to the container. ### IPAM: IP address allocation * `dhcp`: Runs a daemon on the host to make DHCP requests on behalf of the container * `host-local`: maintains a local database of allocated IPs diff --git a/Vagrantfile b/Vagrantfile index fa8fa3c9..02baaeb2 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -8,14 +8,11 @@ Vagrant.configure(2) do |config| config.vm.provision "shell", inline: <<-SHELL set -e -x -u - apt-get update -y || (sleep 40 && apt-get update -y) - apt-get install -y git - + apt-get install -y git gcc-multilib gcc-mingw-w64 wget -qO- https://storage.googleapis.com/golang/go1.10.linux-amd64.tar.gz | tar -C /usr/local -xz - echo 'export GOPATH=/go' >> /root/.bashrc echo 'export PATH=$PATH:/usr/local/go/bin:$GOPATH/bin' >> /root/.bashrc cd /go/src/github.com/containernetworking/plugins SHELL -end +end \ No newline at end of file diff --git a/build.sh b/build.sh index f55eb117..6db861a5 100755 --- a/build.sh +++ b/build.sh @@ -19,12 +19,20 @@ export GO="${GO:-go}" mkdir -p "${PWD}/bin" -echo "Building plugins" +echo "Building plugins ${GOOS}" PLUGINS="plugins/meta/* plugins/main/* plugins/ipam/* plugins/sample" for d in $PLUGINS; do if [ -d "$d" ]; then plugin="$(basename "$d")" - echo " $plugin" - $GO build -o "${PWD}/bin/$plugin" "$@" "$REPO_PATH"/$d + if [ $plugin == "windows" ] + then + if [ "$GOARCH" == "amd64" ] + then + GOOS=windows . $d/build.sh + fi + else + echo " $plugin" + $GO build -o "${PWD}/bin/$plugin" "$@" "$REPO_PATH"/$d + fi fi done diff --git a/pkg/hns/endpoint_windows.go b/pkg/hns/endpoint_windows.go new file mode 100644 index 00000000..9d9185fa --- /dev/null +++ b/pkg/hns/endpoint_windows.go @@ -0,0 +1,152 @@ +// Copyright 2017 CNI authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hns + +import ( + "fmt" + "net" + "strings" + + "github.com/Microsoft/hcsshim" + "github.com/containernetworking/cni/pkg/types/current" + "github.com/juju/errors" +) + +const ( + pauseContainerNetNS = "none" +) + +// GetSandboxContainerID returns the sandbox ID of this pod +func GetSandboxContainerID(containerID string, netNs string) string { + if len(netNs) != 0 && netNs != pauseContainerNetNS { + splits := strings.SplitN(netNs, ":", 2) + if len(splits) == 2 { + containerID = splits[1] + } + } + + return containerID +} + +// ConstructEndpointName constructs enpointId which is used to identify an endpoint from HNS +// There is a special consideration for netNs name here, which is required for Windows Server 1709 +// containerID is the Id of the container on which the endpoint is worked on +func ConstructEndpointName(containerID string, netNs string, networkName string) string { + return GetSandboxContainerID(containerID, netNs) + "_" + networkName +} + +// DeprovisionEndpoint removes an endpoint from the container by sending a Detach request to HNS +// For shared endpoint, ContainerDetach is used +// for removing the endpoint completely, HotDetachEndpoint is used +func DeprovisionEndpoint(epName string, netns string, containerID string) error { + if len(netns) == 0 { + return nil + } + + hnsEndpoint, err := hcsshim.GetHNSEndpointByName(epName) + if err != nil { + return errors.Annotatef(err, "failed to find HNSEndpoint %s", epName) + } + + if netns != pauseContainerNetNS { + // Shared endpoint removal. Do not remove the endpoint. + hnsEndpoint.ContainerDetach(containerID) + return nil + } + + // Do not consider this as failure, else this would leak endpoints + hcsshim.HotDetachEndpoint(containerID, hnsEndpoint.Id) + + // Do not return error + hnsEndpoint.Delete() + + return nil +} + +type EndpointMakerFunc func() (*hcsshim.HNSEndpoint, error) + +// ProvisionEndpoint provisions an endpoint to a container specified by containerID. +// If an endpoint already exists, the endpoint is reused. +// This call is idempotent +func ProvisionEndpoint(epName string, expectedNetworkId string, containerID string, makeEndpoint EndpointMakerFunc) (*hcsshim.HNSEndpoint, error) { + // check if endpoint already exists + createEndpoint := true + hnsEndpoint, err := hcsshim.GetHNSEndpointByName(epName) + if hnsEndpoint != nil && hnsEndpoint.VirtualNetwork == expectedNetworkId { + createEndpoint = false + } + + if createEndpoint { + if hnsEndpoint != nil { + if _, err = hnsEndpoint.Delete(); err != nil { + return nil, errors.Annotate(err, "failed to delete the stale HNSEndpoint") + } + } + + if hnsEndpoint, err = makeEndpoint(); err != nil { + return nil, errors.Annotate(err, "failed to make a new HNSEndpoint") + } + + if hnsEndpoint, err = hnsEndpoint.Create(); err != nil { + return nil, errors.Annotate(err, "failed to create the new HNSEndpoint") + } + + } + + // hot attach + if err := hcsshim.HotAttachEndpoint(containerID, hnsEndpoint.Id); err != nil { + if hcsshim.ErrComputeSystemDoesNotExist == err { + return hnsEndpoint, nil + } + + return nil, err + } + + return hnsEndpoint, nil +} + +// ConstructResult constructs the CNI result for the endpoint +func ConstructResult(hnsNetwork *hcsshim.HNSNetwork, hnsEndpoint *hcsshim.HNSEndpoint) (*current.Result, error) { + resultInterface := ¤t.Interface{ + Name: hnsEndpoint.Name, + Mac: hnsEndpoint.MacAddress, + } + _, ipSubnet, err := net.ParseCIDR(hnsNetwork.Subnets[0].AddressPrefix) + if err != nil { + return nil, errors.Annotatef(err, "failed to parse CIDR from %s", hnsNetwork.Subnets[0].AddressPrefix) + } + + var ipVersion string + if ipv4 := hnsEndpoint.IPAddress.To4(); ipv4 != nil { + ipVersion = "4" + } else if ipv6 := hnsEndpoint.IPAddress.To16(); ipv6 != nil { + ipVersion = "6" + } else { + return nil, fmt.Errorf("IPAddress of HNSEndpoint %s isn't a valid ipv4 or ipv6 Address", hnsEndpoint.Name) + } + + resultIPConfig := ¤t.IPConfig{ + Version: ipVersion, + Address: net.IPNet{ + IP: hnsEndpoint.IPAddress, + Mask: ipSubnet.Mask}, + Gateway: net.ParseIP(hnsEndpoint.GatewayAddress), + } + result := ¤t.Result{} + result.Interfaces = []*current.Interface{resultInterface} + result.IPs = []*current.IPConfig{resultIPConfig} + + return result, nil +} diff --git a/pkg/hns/netconf.go b/pkg/hns/netconf.go new file mode 100644 index 00000000..003ffde9 --- /dev/null +++ b/pkg/hns/netconf.go @@ -0,0 +1,152 @@ +// Copyright 2017 CNI authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hns + +import ( + "encoding/json" + "strings" + + "bytes" + "github.com/buger/jsonparser" + "github.com/containernetworking/cni/pkg/types" +) + +// NetConf is the CNI spec +type NetConf struct { + types.NetConf + + Policies []policy `json:"policies,omitempty"` +} + +type policy struct { + Name string `json:"name"` + Value json.RawMessage `json:"value"` +} + +// MarshalPolicies converts the Endpoint policies in Policies +// to HNS specific policies as Json raw bytes +func (n *NetConf) MarshalPolicies() []json.RawMessage { + if n.Policies == nil { + n.Policies = make([]policy, 0) + } + + result := make([]json.RawMessage, 0, len(n.Policies)) + for _, p := range n.Policies { + if !strings.EqualFold(p.Name, "EndpointPolicy") { + continue + } + + result = append(result, p.Value) + } + + return result +} + +// ApplyOutboundNatPolicy applies NAT Policy in VFP using HNS +// Simultaneously an exception is added for the network that has to be Nat'd +func (n *NetConf) ApplyOutboundNatPolicy(nwToNat string) { + if n.Policies == nil { + n.Policies = make([]policy, 0) + } + + nwToNatBytes := []byte(nwToNat) + + for i, p := range n.Policies { + if !strings.EqualFold(p.Name, "EndpointPolicy") { + continue + } + + typeValue, err := jsonparser.GetUnsafeString(p.Value, "Type") + if err != nil || len(typeValue) == 0 { + continue + } + + if !strings.EqualFold(typeValue, "OutBoundNAT") { + continue + } + + exceptionListValue, dt, _, _ := jsonparser.Get(p.Value, "ExceptionList") + // OutBoundNAT must with ExceptionList, so don't need to judge jsonparser.NotExist + if dt == jsonparser.Array { + buf := bytes.Buffer{} + buf.WriteString(`{"Type": "OutBoundNAT", "ExceptionList": [`) + + jsonparser.ArrayEach(exceptionListValue, func(value []byte, dataType jsonparser.ValueType, offset int, err error) { + if dataType == jsonparser.String && len(value) != 0 { + if bytes.Compare(value, nwToNatBytes) != 0 { + buf.WriteByte('"') + buf.Write(value) + buf.WriteByte('"') + buf.WriteByte(',') + } + } + }) + + buf.WriteString(`"` + nwToNat + `"]}`) + + n.Policies[i] = policy{ + Name: "EndpointPolicy", + Value: buf.Bytes(), + } + } else { + n.Policies[i] = policy{ + Name: "EndpointPolicy", + Value: []byte(`{"Type": "OutBoundNAT", "ExceptionList": ["` + nwToNat + `"]}`), + } + } + + return + } + + // didn't find the policyArg, add it + n.Policies = append(n.Policies, policy{ + Name: "EndpointPolicy", + Value: []byte(`{"Type": "OutBoundNAT", "ExceptionList": ["` + nwToNat + `"]}`), + }) +} + +// ApplyDefaultPAPolicy is used to configure a endpoint PA policy in HNS +func (n *NetConf) ApplyDefaultPAPolicy(paAddress string) { + if n.Policies == nil { + n.Policies = make([]policy, 0) + } + + // if its already present, leave untouched + for i, p := range n.Policies { + if !strings.EqualFold(p.Name, "EndpointPolicy") { + continue + } + + paValue, dt, _, _ := jsonparser.Get(p.Value, "PA") + if dt == jsonparser.NotExist { + continue + } else if dt == jsonparser.String && len(paValue) != 0 { + // found it, don't override + return + } + + n.Policies[i] = policy{ + Name: "EndpointPolicy", + Value: []byte(`{"Type": "PA", "PA": "` + paAddress + `"}`), + } + return + } + + // didn't find the policyArg, add it + n.Policies = append(n.Policies, policy{ + Name: "EndpointPolicy", + Value: []byte(`{"Type": "PA", "PA": "` + paAddress + `"}`), + }) +} diff --git a/pkg/hns/netconf_suite_test.go b/pkg/hns/netconf_suite_test.go new file mode 100644 index 00000000..cc69e6fd --- /dev/null +++ b/pkg/hns/netconf_suite_test.go @@ -0,0 +1,26 @@ +// Copyright 2017 CNI authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package hns + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "testing" +) + +func TestHns(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "HNS NetConf Suite") +} diff --git a/pkg/hns/netconf_test.go b/pkg/hns/netconf_test.go new file mode 100644 index 00000000..0108056f --- /dev/null +++ b/pkg/hns/netconf_test.go @@ -0,0 +1,189 @@ +// Copyright 2017 CNI authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package hns + +import ( + "encoding/json" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +var _ = Describe("HNS NetConf", func() { + Describe("ApplyOutBoundNATPolicy", func() { + Context("when not set by user", func() { + It("sets it by adding a policy", func() { + + // apply it + n := NetConf{} + n.ApplyOutboundNatPolicy("192.168.0.0/16") + + addlArgs := n.Policies + Expect(addlArgs).Should(HaveLen(1)) + + policy := addlArgs[0] + Expect(policy.Name).Should(Equal("EndpointPolicy")) + + value := make(map[string]interface{}) + json.Unmarshal(policy.Value, &value) + + Expect(value).Should(HaveKey("Type")) + Expect(value).Should(HaveKey("ExceptionList")) + Expect(value["Type"]).Should(Equal("OutBoundNAT")) + + exceptionList := value["ExceptionList"].([]interface{}) + Expect(exceptionList).Should(HaveLen(1)) + Expect(exceptionList[0].(string)).Should(Equal("192.168.0.0/16")) + }) + }) + + Context("when set by user", func() { + It("appends exceptions to the existing policy", func() { + // first set it + n := NetConf{} + n.ApplyOutboundNatPolicy("192.168.0.0/16") + + // then attempt to update it + n.ApplyOutboundNatPolicy("10.244.0.0/16") + + // it should be unchanged! + addlArgs := n.Policies + Expect(addlArgs).Should(HaveLen(1)) + + policy := addlArgs[0] + Expect(policy.Name).Should(Equal("EndpointPolicy")) + + var value map[string]interface{} + json.Unmarshal(policy.Value, &value) + + Expect(value).Should(HaveKey("Type")) + Expect(value).Should(HaveKey("ExceptionList")) + Expect(value["Type"]).Should(Equal("OutBoundNAT")) + + exceptionList := value["ExceptionList"].([]interface{}) + Expect(exceptionList).Should(HaveLen(2)) + Expect(exceptionList[0].(string)).Should(Equal("192.168.0.0/16")) + Expect(exceptionList[1].(string)).Should(Equal("10.244.0.0/16")) + }) + }) + }) + + Describe("ApplyDefaultPAPolicy", func() { + Context("when not set by user", func() { + It("sets it by adding a policy", func() { + + n := NetConf{} + n.ApplyDefaultPAPolicy("192.168.0.1") + + addlArgs := n.Policies + Expect(addlArgs).Should(HaveLen(1)) + + policy := addlArgs[0] + Expect(policy.Name).Should(Equal("EndpointPolicy")) + + value := make(map[string]interface{}) + json.Unmarshal(policy.Value, &value) + + Expect(value).Should(HaveKey("Type")) + Expect(value["Type"]).Should(Equal("PA")) + + paAddress := value["PA"].(string) + Expect(paAddress).Should(Equal("192.168.0.1")) + }) + }) + + Context("when set by user", func() { + It("does not override", func() { + n := NetConf{} + n.ApplyDefaultPAPolicy("192.168.0.1") + n.ApplyDefaultPAPolicy("192.168.0.2") + + addlArgs := n.Policies + Expect(addlArgs).Should(HaveLen(1)) + + policy := addlArgs[0] + Expect(policy.Name).Should(Equal("EndpointPolicy")) + + value := make(map[string]interface{}) + json.Unmarshal(policy.Value, &value) + + Expect(value).Should(HaveKey("Type")) + Expect(value["Type"]).Should(Equal("PA")) + + paAddress := value["PA"].(string) + Expect(paAddress).Should(Equal("192.168.0.1")) + Expect(paAddress).ShouldNot(Equal("192.168.0.2")) + }) + }) + }) + + Describe("MarshalPolicies", func() { + Context("when not set by user", func() { + It("sets it by adding a policy", func() { + + n := NetConf{ + Policies: []policy{ + { + Name: "EndpointPolicy", + Value: []byte(`{"someKey": "someValue"}`), + }, + { + Name: "someOtherType", + Value: []byte(`{"someOtherKey": "someOtherValue"}`), + }, + }, + } + + result := n.MarshalPolicies() + Expect(len(result)).To(Equal(1)) + + policy := make(map[string]interface{}) + err := json.Unmarshal(result[0], &policy) + Expect(err).ToNot(HaveOccurred()) + Expect(policy).Should(HaveKey("someKey")) + Expect(policy["someKey"]).To(Equal("someValue")) + }) + }) + + Context("when set by user", func() { + It("appends exceptions to the existing policy", func() { + // first set it + n := NetConf{} + n.ApplyOutboundNatPolicy("192.168.0.0/16") + + // then attempt to update it + n.ApplyOutboundNatPolicy("10.244.0.0/16") + + // it should be unchanged! + addlArgs := n.Policies + Expect(addlArgs).Should(HaveLen(1)) + + policy := addlArgs[0] + Expect(policy.Name).Should(Equal("EndpointPolicy")) + + var value map[string]interface{} + json.Unmarshal(policy.Value, &value) + + Expect(value).Should(HaveKey("Type")) + Expect(value).Should(HaveKey("ExceptionList")) + Expect(value["Type"]).Should(Equal("OutBoundNAT")) + + exceptionList := value["ExceptionList"].([]interface{}) + Expect(exceptionList).Should(HaveLen(2)) + Expect(exceptionList[0].(string)).Should(Equal("192.168.0.0/16")) + Expect(exceptionList[1].(string)).Should(Equal("10.244.0.0/16")) + }) + }) + }) +}) diff --git a/plugins/linux_only.txt b/plugins/linux_only.txt deleted file mode 100644 index 6b04296f..00000000 --- a/plugins/linux_only.txt +++ /dev/null @@ -1,11 +0,0 @@ -plugins/ipam/dhcp -plugins/main/bridge -plugins/main/host-device -plugins/main/ipvlan -plugins/main/loopback -plugins/main/macvlan -plugins/main/ptp -plugins/main/vlan -plugins/meta/portmap -plugins/meta/tuning -plugins/meta/bandwidth \ No newline at end of file diff --git a/plugins/main/windows/build.sh b/plugins/main/windows/build.sh new file mode 100755 index 00000000..d6bcd57f --- /dev/null +++ b/plugins/main/windows/build.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +set -e + +PLUGINS=$(cat plugins/windows_only.txt) +for d in $PLUGINS; do + if [ -d "$d" ]; then + plugin="$(basename "$d").exe" + + echo " $plugin" + CXX=x86_64-w64-mingw32-g++ CC=x86_64-w64-mingw32-gcc CGO_ENABLED=1 \ + $GO build -o "${PWD}/bin/$plugin" "$@" "$REPO_PATH"/$d + fi +done diff --git a/plugins/main/windows/win-bridge/README.md b/plugins/main/windows/win-bridge/README.md new file mode 100644 index 00000000..740410e9 --- /dev/null +++ b/plugins/main/windows/win-bridge/README.md @@ -0,0 +1,25 @@ +# win-bridge plugin + +## Overview + +With win-bridge plugin, all containers (on the same host) are plugged into an L2Bridge network that has one endpoint in the host namespace. + +## Example configuration +``` +{ + "name": "mynet", + "type": "win-bridge", + "ipMasqNetwork": "10.244.0.0/16", + "ipam": { + "type": "host-local", + "subnet": "10.10.0.0/16" + } +} +``` + +## Network configuration reference + +* `name` (string, required): the name of the network. +* `type` (string, required): "win-bridge". +* `ipMasqNetwork` (string, optional): setup NAT if not empty. +* `ipam` (dictionary, required): IPAM configuration to be used for this network. diff --git a/plugins/main/windows/win-bridge/sample.conf b/plugins/main/windows/win-bridge/sample.conf new file mode 100755 index 00000000..319d555e --- /dev/null +++ b/plugins/main/windows/win-bridge/sample.conf @@ -0,0 +1,44 @@ +{ + "name":"cbr0", + "type":"flannel", + "delegate":{ + "type":"win-bridge", + "dns":{ + "nameservers":[ + "11.0.0.10" + ], + "search":[ + "svc.cluster.local" + ] + }, + "policies":[ + { + "name":"EndpointPolicy", + "value":{ + "Type":"OutBoundNAT", + "ExceptionList":[ + "192.168.0.0/16", + "11.0.0.0/8", + "10.137.196.0/23" + ] + } + }, + { + "name":"EndpointPolicy", + "value":{ + "Type":"ROUTE", + "DestinationPrefix":"11.0.0.0/8", + "NeedEncap":true + } + }, + { + "name":"EndpointPolicy", + "value":{ + "Type":"ROUTE", + "DestinationPrefix":"10.137.198.27/32", + "NeedEncap":true + } + } + ] + } +} \ No newline at end of file diff --git a/plugins/main/windows/win-bridge/win-bridge_windows.go b/plugins/main/windows/win-bridge/win-bridge_windows.go new file mode 100644 index 00000000..13439742 --- /dev/null +++ b/plugins/main/windows/win-bridge/win-bridge_windows.go @@ -0,0 +1,151 @@ +// Copyright 2017 CNI authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "encoding/json" + "fmt" + "runtime" + "strings" + + "github.com/juju/errors" + + "github.com/Microsoft/hcsshim" + "github.com/containernetworking/cni/pkg/skel" + "github.com/containernetworking/cni/pkg/types" + "github.com/containernetworking/cni/pkg/types/current" + "github.com/containernetworking/cni/pkg/version" + "github.com/containernetworking/plugins/pkg/hns" + "github.com/containernetworking/plugins/pkg/ipam" +) + +type NetConf struct { + hns.NetConf + + IPMasqNetwork string `json:"ipMasqNetwork,omitempty"` +} + +func init() { + // this ensures that main runs only on main thread (thread group leader). + // since namespace ops (unshare, setns) are done for a single thread, we + // must ensure that the goroutine does not jump from OS thread to thread + runtime.LockOSThread() +} + +func loadNetConf(bytes []byte) (*NetConf, string, error) { + n := &NetConf{} + if err := json.Unmarshal(bytes, n); err != nil { + return nil, "", fmt.Errorf("failed to load netconf: %v", err) + } + return n, n.CNIVersion, nil +} + +func cmdAdd(args *skel.CmdArgs) error { + n, cniVersion, err := loadNetConf(args.StdinData) + if err != nil { + return errors.Annotate(err, "error while loadNetConf") + } + + networkName := n.Name + hnsNetwork, err := hcsshim.GetHNSNetworkByName(networkName) + if err != nil { + return errors.Annotatef(err, "error while GETHNSNewtorkByName(%s)", networkName) + } + + if hnsNetwork == nil { + return fmt.Errorf("network %v not found", networkName) + } + + if !strings.EqualFold(hnsNetwork.Type, "L2Bridge") { + return fmt.Errorf("network %v is of an unexpected type: %v", networkName, hnsNetwork.Type) + } + + epName := hns.ConstructEndpointName(args.ContainerID, args.Netns, n.Name) + + hnsEndpoint, err := hns.ProvisionEndpoint(epName, hnsNetwork.Id, args.ContainerID, func() (*hcsshim.HNSEndpoint, error) { + // run the IPAM plugin and get back the config to apply + r, err := ipam.ExecAdd(n.IPAM.Type, args.StdinData) + if err != nil { + return nil, errors.Annotatef(err, "error while ipam.ExecAdd") + } + + // Convert whatever the IPAM result was into the current Result type + result, err := current.NewResultFromResult(r) + if err != nil { + return nil, errors.Annotatef(err, "error while NewResultFromResult") + } + + if len(result.IPs) == 0 { + return nil, errors.New("IPAM plugin return is missing IP config") + } + + // Calculate gateway for bridge network (needs to be x.2) + gw := result.IPs[0].Address.IP.Mask(result.IPs[0].Address.Mask) + gw[len(gw)-1] += 2 + + // NAT based on the the configured cluster network + if len(n.IPMasqNetwork) != 0 { + n.ApplyOutboundNatPolicy(n.IPMasqNetwork) + } + + result.DNS = n.DNS + + hnsEndpoint := &hcsshim.HNSEndpoint{ + Name: epName, + VirtualNetwork: hnsNetwork.Id, + DNSServerList: strings.Join(result.DNS.Nameservers, ","), + DNSSuffix: strings.Join(result.DNS.Search, ","), + GatewayAddress: gw.String(), + IPAddress: result.IPs[0].Address.IP, + Policies: n.MarshalPolicies(), + } + + return hnsEndpoint, nil + }) + if err != nil { + return errors.Annotatef(err, "error while ProvisionEndpoint(%v,%v,%v)", epName, hnsNetwork.Id, args.ContainerID) + } + + result, err := hns.ConstructResult(hnsNetwork, hnsEndpoint) + if err != nil { + return errors.Annotatef(err, "error while constructResult") + } + + return types.PrintResult(result, cniVersion) +} + +func cmdDel(args *skel.CmdArgs) error { + n, _, err := loadNetConf(args.StdinData) + if err != nil { + return err + } + + if err := ipam.ExecDel(n.IPAM.Type, args.StdinData); err != nil { + return err + } + + epName := hns.ConstructEndpointName(args.ContainerID, args.Netns, n.Name) + + return hns.DeprovisionEndpoint(epName, args.Netns, args.ContainerID) +} + +func cmdGet(_ *skel.CmdArgs) error { + // TODO: implement + return fmt.Errorf("not implemented") +} + +func main() { + skel.PluginMain(cmdAdd, cmdGet, cmdDel, version.All, "TODO") +} diff --git a/plugins/main/windows/win-overlay/README.md b/plugins/main/windows/win-overlay/README.md new file mode 100644 index 00000000..4c15ff25 --- /dev/null +++ b/plugins/main/windows/win-overlay/README.md @@ -0,0 +1,27 @@ +# win-overlay plugin + +## Overview + +With win-overlay plugin, all containers (on the same host) are plugged into an Overlay network based on VXLAN encapsulation. + +## Example configuration +``` +{ + "name": "mynet", + "type": "win-overlay", + "ipMasq": true, + "endpointMacPrefix": "0E-2A", + "ipam": { + "type": "host-local", + "subnet": "10.10.0.0/16" + } +} +``` + +## Network configuration reference + +* `name` (string, required): the name of the network. +* `type` (string, required): "win-overlay". +* `ipMasq` (bool, optional): the inverse of `$FLANNEL_IPMASQ`, setup NAT for the hnsNetwork subnet. +* `endpointMacPrefix` (string, optional): set to the MAC prefix configured for Flannel +* `ipam` (dictionary, required): IPAM configuration to be used for this network. diff --git a/plugins/main/windows/win-overlay/sample.conf b/plugins/main/windows/win-overlay/sample.conf new file mode 100755 index 00000000..fcb6a2fa --- /dev/null +++ b/plugins/main/windows/win-overlay/sample.conf @@ -0,0 +1,36 @@ +{ + "cniVersion":"0.2.0", + "name":"vxlan0", + "type":"flannel", + "delegate":{ + "type":"win-overlay", + "dns":{ + "nameservers":[ + "11.0.0.10" + ], + "search":[ + "svc.cluster.local" + ] + }, + "policies":[ + { + "name":"EndpointPolicy", + "value":{ + "Type":"OutBoundNAT", + "ExceptionList":[ + "192.168.0.0/16", + "11.0.0.0/8" + ] + } + }, + { + "name":"EndpointPolicy", + "value":{ + "Type":"ROUTE", + "DestinationPrefix":"11.0.0.0/8", + "NeedEncap":true + } + } + ] + } +} \ No newline at end of file diff --git a/plugins/main/windows/win-overlay/win-overlay_windows.go b/plugins/main/windows/win-overlay/win-overlay_windows.go new file mode 100644 index 00000000..cefe68a6 --- /dev/null +++ b/plugins/main/windows/win-overlay/win-overlay_windows.go @@ -0,0 +1,166 @@ +// Copyright 2017 CNI authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "encoding/json" + "fmt" + "runtime" + "strings" + + "github.com/juju/errors" + + "github.com/Microsoft/hcsshim" + "github.com/containernetworking/cni/pkg/skel" + "github.com/containernetworking/cni/pkg/types" + "github.com/containernetworking/cni/pkg/types/current" + "github.com/containernetworking/cni/pkg/version" + "github.com/containernetworking/plugins/pkg/hns" + "github.com/containernetworking/plugins/pkg/ipam" +) + +type NetConf struct { + hns.NetConf + + IPMasq bool `json:"ipMasq"` + EndpointMacPrefix string `json:"endpointMacPrefix,omitempty"` +} + +func init() { + // this ensures that main runs only on main thread (thread group leader). + // since namespace ops (unshare, setns) are done for a single thread, we + // must ensure that the goroutine does not jump from OS thread to thread + runtime.LockOSThread() +} + +func loadNetConf(bytes []byte) (*NetConf, string, error) { + n := &NetConf{} + if err := json.Unmarshal(bytes, n); err != nil { + return nil, "", fmt.Errorf("failed to load netconf: %v", err) + } + return n, n.CNIVersion, nil +} + +func cmdAdd(args *skel.CmdArgs) error { + n, cniVersion, err := loadNetConf(args.StdinData) + if err != nil { + return errors.Annotate(err, "error while loadNetConf") + } + + if len(n.EndpointMacPrefix) != 0 { + if len(n.EndpointMacPrefix) != 5 || n.EndpointMacPrefix[2] != '-' { + return fmt.Errorf("endpointMacPrefix [%v] is invalid, value must be of the format xx-xx", n.EndpointMacPrefix) + } + } else { + n.EndpointMacPrefix = "0E-2A" + } + + networkName := n.Name + hnsNetwork, err := hcsshim.GetHNSNetworkByName(networkName) + if err != nil { + return errors.Annotatef(err, "error while GETHNSNewtorkByName(%s)", networkName) + } + + if hnsNetwork == nil { + return fmt.Errorf("network %v not found", networkName) + } + + if !strings.EqualFold(hnsNetwork.Type, "Overlay") { + return fmt.Errorf("network %v is of an unexpected type: %v", networkName, hnsNetwork.Type) + } + + epName := hns.ConstructEndpointName(args.ContainerID, args.Netns, n.Name) + + hnsEndpoint, err := hns.ProvisionEndpoint(epName, hnsNetwork.Id, args.ContainerID, func() (*hcsshim.HNSEndpoint, error) { + // run the IPAM plugin and get back the config to apply + r, err := ipam.ExecAdd(n.IPAM.Type, args.StdinData) + if err != nil { + return nil, errors.Annotatef(err, "error while ipam.ExecAdd") + } + + // Convert whatever the IPAM result was into the current Result type + result, err := current.NewResultFromResult(r) + if err != nil { + return nil, errors.Annotatef(err, "error while NewResultFromResult") + } + + if len(result.IPs) == 0 { + return nil, errors.New("IPAM plugin return is missing IP config") + } + + ipAddr := result.IPs[0].Address.IP.To4() + if ipAddr == nil { + return nil, errors.New("win-overlay doesn't support IPv6 now") + } + + // conjure a MAC based on the IP for Overlay + macAddr := fmt.Sprintf("%v-%02x-%02x-%02x-%02x", n.EndpointMacPrefix, ipAddr[0], ipAddr[1], ipAddr[2], ipAddr[3]) + // use the HNS network gateway + gw := hnsNetwork.Subnets[0].GatewayAddress + n.ApplyDefaultPAPolicy(hnsNetwork.ManagementIP) + if n.IPMasq { + n.ApplyOutboundNatPolicy(hnsNetwork.Subnets[0].AddressPrefix) + } + + result.DNS = n.DNS + + hnsEndpoint := &hcsshim.HNSEndpoint{ + Name: epName, + VirtualNetwork: hnsNetwork.Id, + DNSServerList: strings.Join(result.DNS.Nameservers, ","), + DNSSuffix: strings.Join(result.DNS.Search, ","), + GatewayAddress: gw, + IPAddress: ipAddr, + MacAddress: macAddr, + Policies: n.MarshalPolicies(), + } + + return hnsEndpoint, nil + }) + if err != nil { + return errors.Annotatef(err, "error while ProvisionEndpoint(%v,%v,%v)", epName, hnsNetwork.Id, args.ContainerID) + } + + result, err := hns.ConstructResult(hnsNetwork, hnsEndpoint) + if err != nil { + return errors.Annotatef(err, "error while constructResult") + } + + return types.PrintResult(result, cniVersion) +} + +func cmdDel(args *skel.CmdArgs) error { + n, _, err := loadNetConf(args.StdinData) + if err != nil { + return err + } + + if err := ipam.ExecDel(n.IPAM.Type, args.StdinData); err != nil { + return err + } + + epName := hns.ConstructEndpointName(args.ContainerID, args.Netns, n.Name) + + return hns.DeprovisionEndpoint(epName, args.Netns, args.ContainerID) +} + +func cmdGet(_ *skel.CmdArgs) error { + // TODO: implement + return fmt.Errorf("not implemented") +} + +func main() { + skel.PluginMain(cmdAdd, cmdGet, cmdDel, version.All, "TODO") +} diff --git a/plugins/meta/flannel/README.md b/plugins/meta/flannel/README.md index 0efb6905..da7961b9 100644 --- a/plugins/meta/flannel/README.md +++ b/plugins/meta/flannel/README.md @@ -86,3 +86,50 @@ flannel plugin will set the following fields in the delegated plugin configurati * `mtu`: `$FLANNEL_MTU` Additionally, for the bridge plugin, `isGateway` will be set to `true`, if not present. + +## Windows Support (Experimental) +This plugin supports delegating to the windows CNI plugins (overlay.exe, l2bridge.exe) to work in conjunction with [Flannel on Windows](https://github.com/coreos/flannel/issues/833). +Flannel sets up an [HNS Network](https://docs.microsoft.com/en-us/virtualization/windowscontainers/manage-containers/container-networking) in L2Bridge mode for host-gw and in Overlay mode for vxlan. + +The following fields must be set in the delegated plugin configuration: +* `name` (string, required): the name of the network (must match the name in Flannel config / name of the HNS network) +* `type` (string, optional): set to `win-l2bridge` by default. Can be set to `win-overlay` or other custom windows CNI +* `ipMasq`: the inverse of `$FLANNEL_IPMASQ` +* `endpointMacPrefix` (string, optional): required for `win-overlay` mode, set to the MAC prefix configured for Flannel +* `clusterNetworkPrefix` (string, optional): required for `win-l2bridge` mode, setup NAT if `ipMasq` is set to true + +For `win-l2bridge`, the Flannel CNI plugin will set: +* `ipam`: "host-local" type will be used with "subnet" set to `$FLANNEL_SUBNET` and gateway as the .2 address in `$FLANNEL_NETWORK` + +For `win-overlay`, the Flannel CNI plugin will set: +* `ipam`: "host-local" type will be used with "subnet" set to `$FLANNEL_SUBNET` and gateway as the .1 address in `$FLANNEL_NETWORK` + +If IPMASQ is true, the Flannel CNI plugin will setup an OutBoundNAT policy and add FLANNEL_SUBNET to any existing exclusions. + +All other delegate config e.g. other HNS endpoint policies in AdditionalArgs will be passed to WINCNI as-is. + +Example VXLAN Flannel CNI config +``` +{ + "name": "mynet", + "type": "flannel", + "delegate": { + "type": "win-overlay", + "endpointMacPrefix": "0E-2A" + } +} +``` + +For this example, Flannel CNI would generate the following config to delegate to the windows CNI when FLANNEL_NETWORK=10.244.0.0/16, FLANNEL_SUBNET=10.244.1.0/24 and IPMASQ=true +``` +{ + "name": "mynet", + "type": "win-overlay", + "endpointMacPrefix": "0E-2A", + "ipMasq": true, + "ipam": { + "subnet": "10.244.1.0/24", + "type": "host-local" + } +} +``` \ No newline at end of file diff --git a/plugins/meta/flannel/flannel.go b/plugins/meta/flannel/flannel.go index 21190281..f7d1b957 100644 --- a/plugins/meta/flannel/flannel.go +++ b/plugins/meta/flannel/flannel.go @@ -42,6 +42,7 @@ const ( type NetConf struct { types.NetConf + SubnetFile string `json:"subnetFile"` DataDir string `json:"dataDir"` Delegate map[string]interface{} `json:"delegate"` @@ -202,43 +203,7 @@ func cmdAdd(args *skel.CmdArgs) error { } } - n.Delegate["name"] = n.Name - - if !hasKey(n.Delegate, "type") { - n.Delegate["type"] = "bridge" - } - - if !hasKey(n.Delegate, "ipMasq") { - // if flannel is not doing ipmasq, we should - ipmasq := !*fenv.ipmasq - n.Delegate["ipMasq"] = ipmasq - } - - if !hasKey(n.Delegate, "mtu") { - mtu := fenv.mtu - n.Delegate["mtu"] = mtu - } - - if n.Delegate["type"].(string) == "bridge" { - if !hasKey(n.Delegate, "isGateway") { - n.Delegate["isGateway"] = true - } - } - if n.CNIVersion != "" { - n.Delegate["cniVersion"] = n.CNIVersion - } - - n.Delegate["ipam"] = map[string]interface{}{ - "type": "host-local", - "subnet": fenv.sn.String(), - "routes": []types.Route{ - types.Route{ - Dst: *fenv.nw, - }, - }, - } - - return delegateAdd(args.ContainerID, n.DataDir, n.Delegate) + return doCmdAdd(args, n, fenv) } func cmdDel(args *skel.CmdArgs) error { @@ -247,25 +212,10 @@ func cmdDel(args *skel.CmdArgs) error { return err } - netconfBytes, err := consumeScratchNetConf(args.ContainerID, nc.DataDir) - if err != nil { - if os.IsNotExist(err) { - // Per spec should ignore error if resources are missing / already removed - return nil - } - return err - } - - n := &types.NetConf{} - if err = json.Unmarshal(netconfBytes, n); err != nil { - return fmt.Errorf("failed to parse netconf: %v", err) - } - - return invoke.DelegateDel(n.Type, netconfBytes, nil) + return doCmdDel(args, nc) } func main() { - // TODO: implement plugin version skel.PluginMain(cmdAdd, cmdGet, cmdDel, version.All, "TODO") } diff --git a/plugins/meta/flannel/flannel_linux.go b/plugins/meta/flannel/flannel_linux.go new file mode 100644 index 00000000..f89a5549 --- /dev/null +++ b/plugins/meta/flannel/flannel_linux.go @@ -0,0 +1,86 @@ +// Copyright 2018 CNI authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This is a "meta-plugin". It reads in its own netconf, combines it with +// the data from flannel generated subnet file and then invokes a plugin +// like bridge or ipvlan to do the real work. + +package main + +import ( + "encoding/json" + "fmt" + "github.com/containernetworking/cni/pkg/invoke" + "github.com/containernetworking/cni/pkg/skel" + "github.com/containernetworking/cni/pkg/types" + "os" +) + +func doCmdAdd(args *skel.CmdArgs, n *NetConf, fenv *subnetEnv) error { + n.Delegate["name"] = n.Name + + if !hasKey(n.Delegate, "type") { + n.Delegate["type"] = "bridge" + } + + if !hasKey(n.Delegate, "ipMasq") { + // if flannel is not doing ipmasq, we should + ipmasq := !*fenv.ipmasq + n.Delegate["ipMasq"] = ipmasq + } + + if !hasKey(n.Delegate, "mtu") { + mtu := fenv.mtu + n.Delegate["mtu"] = mtu + } + + if n.Delegate["type"].(string) == "bridge" { + if !hasKey(n.Delegate, "isGateway") { + n.Delegate["isGateway"] = true + } + } + if n.CNIVersion != "" { + n.Delegate["cniVersion"] = n.CNIVersion + } + + n.Delegate["ipam"] = map[string]interface{}{ + "type": "host-local", + "subnet": fenv.sn.String(), + "routes": []types.Route{ + { + Dst: *fenv.nw, + }, + }, + } + + return delegateAdd(args.ContainerID, n.DataDir, n.Delegate) +} + +func doCmdDel(args *skel.CmdArgs, n *NetConf) error { + netconfBytes, err := consumeScratchNetConf(args.ContainerID, n.DataDir) + if err != nil { + if os.IsNotExist(err) { + // Per spec should ignore error if resources are missing / already removed + return nil + } + return err + } + + nc := &types.NetConf{} + if err = json.Unmarshal(netconfBytes, nc); err != nil { + return fmt.Errorf("failed to parse netconf: %v", err) + } + + return invoke.DelegateDel(nc.Type, netconfBytes, nil) +} diff --git a/plugins/meta/flannel/flannel_windows.go b/plugins/meta/flannel/flannel_windows.go new file mode 100644 index 00000000..d8cacffe --- /dev/null +++ b/plugins/meta/flannel/flannel_windows.go @@ -0,0 +1,73 @@ +// Copyright 2018 CNI authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This is a "meta-plugin". It reads in its own netconf, combines it with +// the data from flannel generated subnet file and then invokes a plugin +// like bridge or ipvlan to do the real work. + +package main + +import ( + "encoding/json" + "fmt" + "github.com/containernetworking/cni/pkg/invoke" + "github.com/containernetworking/cni/pkg/skel" + "github.com/containernetworking/cni/pkg/types" + "github.com/containernetworking/cni/pkg/types/020" + "github.com/containernetworking/plugins/pkg/hns" + "os" +) + +func doCmdAdd(args *skel.CmdArgs, n *NetConf, fenv *subnetEnv) error { + n.Delegate["name"] = n.Name + + if !hasKey(n.Delegate, "type") { + n.Delegate["type"] = "win-bridge" + } + + // if flannel needs ipmasq - get the plugin to configure it + // (this is the opposite of how linux works - on linux the flannel daemon configure ipmasq) + n.Delegate["ipMasq"] = *fenv.ipmasq + n.Delegate["ipMasqNetwork"] = fenv.nw.String() + + n.Delegate["cniVersion"] = types020.ImplementedSpecVersion + if len(n.CNIVersion) != 0 { + n.Delegate["cniVersion"] = n.CNIVersion + } + + n.Delegate["ipam"] = map[string]interface{}{ + "type": "host-local", + "subnet": fenv.sn.String(), + } + + return delegateAdd(hns.GetSandboxContainerID(args.ContainerID, args.Netns), n.DataDir, n.Delegate) +} + +func doCmdDel(args *skel.CmdArgs, n *NetConf) error { + netconfBytes, err := consumeScratchNetConf(hns.GetSandboxContainerID(args.ContainerID, args.Netns), n.DataDir) + if err != nil { + if os.IsNotExist(err) { + // Per spec should ignore error if resources are missing / already removed + return nil + } + return err + } + + nc := &types.NetConf{} + if err = json.Unmarshal(netconfBytes, nc); err != nil { + return fmt.Errorf("failed to parse netconf: %v", err) + } + + return invoke.DelegateDel(nc.Type, netconfBytes, nil) +} diff --git a/plugins/windows_only.txt b/plugins/windows_only.txt new file mode 100644 index 00000000..47f31094 --- /dev/null +++ b/plugins/windows_only.txt @@ -0,0 +1,4 @@ +plugins/ipam/host-local +plugins/main/windows/win-bridge +plugins/main/windows/win-overlay +plugins/meta/flannel \ No newline at end of file