1
0
mirror of https://github.com/rancher/os.git synced 2025-09-03 15:54:24 +00:00

Add CNI networking to system-docker

This commit is contained in:
Darren Shepherd
2016-06-28 15:13:45 -07:00
parent 116c147620
commit 0323844ca6
76 changed files with 5603 additions and 35 deletions

View File

@@ -0,0 +1,86 @@
# host-local IP address manager
host-local IPAM allocates IPv4 and IPv6 addresses out of a specified address range.
## Usage
### Obtain an IP
Given the following network configuration:
```
{
"name": "default",
"ipam": {
"type": "host-local",
"subnet": "203.0.113.0/24"
}
}
```
#### Using the command line interface
```
$ export CNI_COMMAND=ADD
$ export CNI_CONTAINERID=f81d4fae-7dec-11d0-a765-00a0c91e6bf6
$ ./host-local < $conf
```
```
{
"ip4": {
"ip": "203.0.113.1/24"
}
}
```
## Backends
By default ipmanager stores IP allocations on the local filesystem using the IP address as the file name and the ID as contents. For example:
```
$ ls /var/lib/cni/networks/default
```
```
203.0.113.1 203.0.113.2
```
```
$ cat /var/lib/cni/networks/default/203.0.113.1
```
```
f81d4fae-7dec-11d0-a765-00a0c91e6bf6
```
## Configuration Files
```
{
"name": "ipv6",
"ipam": {
"type": "host-local",
"subnet": "3ffe:ffff:0:01ff::/64",
"range-start": "3ffe:ffff:0:01ff::0010",
"range-end": "3ffe:ffff:0:01ff::0020",
"routes": [
{ "dst": "3ffe:ffff:0:01ff::1/64" }
]
}
}
```
```
{
"name": "ipv4",
"ipam": {
"type": "host-local",
"subnet": "203.0.113.1/24",
"range-start": "203.0.113.10",
"range-end": "203.0.113.20",
"routes": [
{ "dst": "203.0.113.0/24" }
]
}
}
```

View File

@@ -0,0 +1,202 @@
// Copyright 2015 CNI authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package hostlocal
import (
"fmt"
"log"
"net"
"github.com/containernetworking/cni/pkg/ip"
"github.com/containernetworking/cni/pkg/types"
"github.com/containernetworking/cni/plugins/ipam/host-local/backend"
)
type IPAllocator struct {
start net.IP
end net.IP
conf *IPAMConfig
store backend.Store
}
func NewIPAllocator(conf *IPAMConfig, store backend.Store) (*IPAllocator, error) {
var (
start net.IP
end net.IP
err error
)
start, end, err = networkRange((*net.IPNet)(&conf.Subnet))
if err != nil {
return nil, err
}
// skip the .0 address
start = ip.NextIP(start)
if conf.RangeStart != nil {
if err := validateRangeIP(conf.RangeStart, (*net.IPNet)(&conf.Subnet)); err != nil {
return nil, err
}
start = conf.RangeStart
}
if conf.RangeEnd != nil {
if err := validateRangeIP(conf.RangeEnd, (*net.IPNet)(&conf.Subnet)); err != nil {
return nil, err
}
// RangeEnd is inclusive
end = ip.NextIP(conf.RangeEnd)
}
return &IPAllocator{start, end, conf, store}, nil
}
func validateRangeIP(ip net.IP, ipnet *net.IPNet) error {
if !ipnet.Contains(ip) {
return fmt.Errorf("%s not in network: %s", ip, ipnet)
}
return nil
}
// Returns newly allocated IP along with its config
func (a *IPAllocator) Get(id string) (*types.IPConfig, error) {
a.store.Lock()
defer a.store.Unlock()
gw := a.conf.Gateway
if gw == nil {
gw = ip.NextIP(a.conf.Subnet.IP)
}
var requestedIP net.IP
if a.conf.Args != nil {
requestedIP = a.conf.Args.IP
}
if requestedIP != nil {
if gw != nil && gw.Equal(a.conf.Args.IP) {
return nil, fmt.Errorf("requested IP must differ gateway IP")
}
subnet := net.IPNet{
IP: a.conf.Subnet.IP,
Mask: a.conf.Subnet.Mask,
}
err := validateRangeIP(requestedIP, &subnet)
if err != nil {
return nil, err
}
reserved, err := a.store.Reserve(id, requestedIP)
if err != nil {
return nil, err
}
if reserved {
return &types.IPConfig{
IP: net.IPNet{IP: requestedIP, Mask: a.conf.Subnet.Mask},
Gateway: gw,
Routes: a.conf.Routes,
}, nil
}
return nil, fmt.Errorf("requested IP address %q is not available in network: %s", requestedIP, a.conf.Name)
}
startIP, endIP := a.getSearchRange()
for cur := startIP; !cur.Equal(endIP); cur = a.nextIP(cur) {
// don't allocate gateway IP
if gw != nil && cur.Equal(gw) {
continue
}
reserved, err := a.store.Reserve(id, cur)
if err != nil {
return nil, err
}
if reserved {
return &types.IPConfig{
IP: net.IPNet{IP: cur, Mask: a.conf.Subnet.Mask},
Gateway: gw,
Routes: a.conf.Routes,
}, nil
}
}
return nil, fmt.Errorf("no IP addresses available in network: %s", a.conf.Name)
}
// Releases all IPs allocated for the container with given ID
func (a *IPAllocator) Release(id string) error {
a.store.Lock()
defer a.store.Unlock()
return a.store.ReleaseByID(id)
}
func networkRange(ipnet *net.IPNet) (net.IP, net.IP, error) {
if ipnet.IP == nil {
return nil, nil, fmt.Errorf("missing field %q in IPAM configuration", "subnet")
}
ip := ipnet.IP.To4()
if ip == nil {
ip = ipnet.IP.To16()
if ip == nil {
return nil, nil, fmt.Errorf("IP not v4 nor v6")
}
}
if len(ip) != len(ipnet.Mask) {
return nil, nil, fmt.Errorf("IPNet IP and Mask version mismatch")
}
var end net.IP
for i := 0; i < len(ip); i++ {
end = append(end, ip[i]|^ipnet.Mask[i])
}
return ipnet.IP, end, nil
}
// nextIP returns the next ip of curIP within ipallocator's subnet
func (a *IPAllocator) nextIP(curIP net.IP) net.IP {
if curIP.Equal(a.end) {
return a.start
}
return ip.NextIP(curIP)
}
// getSearchRange returns the start and end ip based on the last reserved ip
func (a *IPAllocator) getSearchRange() (net.IP, net.IP) {
var startIP net.IP
var endIP net.IP
startFromLastReservedIP := false
lastReservedIP, err := a.store.LastReservedIP()
if err != nil {
log.Printf("Error retriving last reserved ip: %v", err)
} else if lastReservedIP != nil {
subnet := net.IPNet{
IP: a.conf.Subnet.IP,
Mask: a.conf.Subnet.Mask,
}
err := validateRangeIP(lastReservedIP, &subnet)
if err == nil {
startFromLastReservedIP = true
}
}
if startFromLastReservedIP {
startIP = a.nextIP(lastReservedIP)
endIP = lastReservedIP
} else {
startIP = a.start
endIP = a.end
}
return startIP, endIP
}

View File

@@ -0,0 +1,107 @@
// Copyright 2015 CNI authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package disk
import (
"fmt"
"io/ioutil"
"net"
"os"
"path/filepath"
)
const lastIPFile = "last_reserved_ip"
var defaultDataDir = "/var/lib/cni/networks"
type Store struct {
FileLock
dataDir string
}
func New(network string) (*Store, error) {
dir := filepath.Join(defaultDataDir, network)
if err := os.MkdirAll(dir, 0644); err != nil {
return nil, err
}
lk, err := NewFileLock(dir)
if err != nil {
return nil, err
}
return &Store{*lk, dir}, nil
}
func (s *Store) Reserve(id string, ip net.IP) (bool, error) {
fname := filepath.Join(s.dataDir, ip.String())
f, err := os.OpenFile(fname, os.O_RDWR|os.O_EXCL|os.O_CREATE, 0644)
if os.IsExist(err) {
return false, nil
}
if err != nil {
return false, err
}
if _, err := f.WriteString(id); err != nil {
f.Close()
os.Remove(f.Name())
return false, err
}
if err := f.Close(); err != nil {
os.Remove(f.Name())
return false, err
}
// store the reserved ip in lastIPFile
ipfile := filepath.Join(s.dataDir, lastIPFile)
err = ioutil.WriteFile(ipfile, []byte(ip.String()), 0644)
if err != nil {
return false, err
}
return true, nil
}
// LastReservedIP returns the last reserved IP if exists
func (s *Store) LastReservedIP() (net.IP, error) {
ipfile := filepath.Join(s.dataDir, lastIPFile)
data, err := ioutil.ReadFile(ipfile)
if err != nil {
return nil, fmt.Errorf("Failed to retrieve last reserved ip: %v", err)
}
return net.ParseIP(string(data)), nil
}
func (s *Store) Release(ip net.IP) error {
return os.Remove(filepath.Join(s.dataDir, ip.String()))
}
// N.B. This function eats errors to be tolerant and
// release as much as possible
func (s *Store) ReleaseByID(id string) error {
err := filepath.Walk(s.dataDir, func(path string, info os.FileInfo, err error) error {
if err != nil || info.IsDir() {
return nil
}
data, err := ioutil.ReadFile(path)
if err != nil {
return nil
}
if string(data) == id {
if err := os.Remove(path); err != nil {
return nil
}
}
return nil
})
return err
}

View File

@@ -0,0 +1,50 @@
// Copyright 2015 CNI authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package disk
import (
"os"
"syscall"
)
// FileLock wraps os.File to be used as a lock using flock
type FileLock struct {
f *os.File
}
// NewFileLock opens file/dir at path and returns unlocked FileLock object
func NewFileLock(path string) (*FileLock, error) {
f, err := os.Open(path)
if err != nil {
return nil, err
}
return &FileLock{f}, nil
}
// Close closes underlying file
func (l *FileLock) Close() error {
return l.f.Close()
}
// Lock acquires an exclusive lock
func (l *FileLock) Lock() error {
return syscall.Flock(int(l.f.Fd()), syscall.LOCK_EX)
}
// Unlock releases the lock
func (l *FileLock) Unlock() error {
return syscall.Flock(int(l.f.Fd()), syscall.LOCK_UN)
}

View File

@@ -0,0 +1,27 @@
// Copyright 2015 CNI authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package backend
import "net"
type Store interface {
Lock() error
Unlock() error
Close() error
Reserve(id string, ip net.IP) (bool, error)
LastReservedIP() (net.IP, error)
Release(ip net.IP) error
ReleaseByID(id string) error
}

View File

@@ -0,0 +1,70 @@
// Copyright 2015 CNI authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package hostlocal
import (
"encoding/json"
"fmt"
"net"
"github.com/containernetworking/cni/pkg/types"
)
// IPAMConfig represents the IP related network configuration.
type IPAMConfig struct {
Name string
Type string `json:"type"`
RangeStart net.IP `json:"rangeStart"`
RangeEnd net.IP `json:"rangeEnd"`
Subnet types.IPNet `json:"subnet"`
Gateway net.IP `json:"gateway"`
Routes []types.Route `json:"routes"`
Args *IPAMArgs `json:"-"`
}
type IPAMArgs struct {
types.CommonArgs
IP net.IP `json:"ip,omitempty"`
}
type Net struct {
Name string `json:"name"`
IPAM *IPAMConfig `json:"ipam"`
}
// NewIPAMConfig creates a NetworkConfig from the given network name.
func LoadIPAMConfig(bytes []byte, args string) (*IPAMConfig, error) {
n := Net{}
if err := json.Unmarshal(bytes, &n); err != nil {
return nil, err
}
if args != "" {
n.IPAM.Args = &IPAMArgs{}
err := types.LoadArgs(args, n.IPAM.Args)
if err != nil {
return nil, err
}
}
if n.IPAM == nil {
return nil, fmt.Errorf("IPAM config missing 'ipam' key")
}
// Copy net name into IPAM so not to drag Net struct around
n.IPAM.Name = n.Name
return n.IPAM, nil
}

View File

@@ -0,0 +1,74 @@
// Copyright 2015 CNI authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package hostlocal
import (
"github.com/containernetworking/cni/plugins/ipam/host-local/backend/disk"
"github.com/containernetworking/cni/pkg/skel"
"github.com/containernetworking/cni/pkg/types"
)
func Main() {
skel.PluginMain(cmdAdd, cmdDel)
}
func cmdAdd(args *skel.CmdArgs) error {
ipamConf, err := LoadIPAMConfig(args.StdinData, args.Args)
if err != nil {
return err
}
store, err := disk.New(ipamConf.Name)
if err != nil {
return err
}
defer store.Close()
allocator, err := NewIPAllocator(ipamConf, store)
if err != nil {
return err
}
ipConf, err := allocator.Get(args.ContainerID)
if err != nil {
return err
}
r := &types.Result{
IP4: ipConf,
}
return r.Print()
}
func cmdDel(args *skel.CmdArgs) error {
ipamConf, err := LoadIPAMConfig(args.StdinData, args.Args)
if err != nil {
return err
}
store, err := disk.New(ipamConf.Name)
if err != nil {
return err
}
defer store.Close()
allocator, err := NewIPAllocator(ipamConf, store)
if err != nil {
return err
}
return allocator.Release(args.ContainerID)
}

View File

@@ -0,0 +1,319 @@
// Copyright 2014 CNI authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package bridge
import (
"encoding/json"
"errors"
"fmt"
"net"
"runtime"
"syscall"
"github.com/containernetworking/cni/pkg/ip"
"github.com/containernetworking/cni/pkg/ipam"
"github.com/containernetworking/cni/pkg/ns"
"github.com/containernetworking/cni/pkg/skel"
"github.com/containernetworking/cni/pkg/types"
"github.com/containernetworking/cni/pkg/utils"
"github.com/vishvananda/netlink"
)
const defaultBrName = "cni0"
type NetConf struct {
types.NetConf
BrName string `json:"bridge"`
IsGW bool `json:"isGateway"`
IsDefaultGW bool `json:"isDefaultGateway"`
IPMasq bool `json:"ipMasq"`
MTU int `json:"mtu"`
HairpinMode bool `json:"hairpinMode"`
}
func init() {
// this ensures that main runs only on main thread (thread group leader).
// since namespace ops (unshare, setns) are done for a single thread, we
// must ensure that the goroutine does not jump from OS thread to thread
runtime.LockOSThread()
}
func loadNetConf(bytes []byte) (*NetConf, error) {
n := &NetConf{
BrName: defaultBrName,
}
if err := json.Unmarshal(bytes, n); err != nil {
return nil, fmt.Errorf("failed to load netconf: %v", err)
}
return n, nil
}
func ensureBridgeAddr(br *netlink.Bridge, ipn *net.IPNet) error {
addrs, err := netlink.AddrList(br, syscall.AF_INET)
if err != nil && err != syscall.ENOENT {
return fmt.Errorf("could not get list of IP addresses: %v", err)
}
// if there're no addresses on the bridge, it's ok -- we'll add one
if len(addrs) > 0 {
ipnStr := ipn.String()
for _, a := range addrs {
// string comp is actually easiest for doing IPNet comps
if a.IPNet.String() == ipnStr {
return nil
}
}
return fmt.Errorf("%q already has an IP address different from %v", br.Name, ipn.String())
}
addr := &netlink.Addr{IPNet: ipn, Label: ""}
if err := netlink.AddrAdd(br, addr); err != nil {
return fmt.Errorf("could not add IP address to %q: %v", br.Name, err)
}
return nil
}
func bridgeByName(name string) (*netlink.Bridge, error) {
l, err := netlink.LinkByName(name)
if err != nil {
return nil, fmt.Errorf("could not lookup %q: %v", name, err)
}
br, ok := l.(*netlink.Bridge)
if !ok {
return nil, fmt.Errorf("%q already exists but is not a bridge", name)
}
return br, nil
}
func ensureBridge(brName string, mtu int) (*netlink.Bridge, error) {
br := &netlink.Bridge{
LinkAttrs: netlink.LinkAttrs{
Name: brName,
MTU: mtu,
// Let kernel use default txqueuelen; leaving it unset
// means 0, and a zero-length TX queue messes up FIFO
// traffic shapers which use TX queue length as the
// default packet limit
TxQLen: -1,
},
}
if err := netlink.LinkAdd(br); err != nil {
if err != syscall.EEXIST {
return nil, fmt.Errorf("could not add %q: %v", brName, err)
}
// it's ok if the device already exists as long as config is similar
br, err = bridgeByName(brName)
if err != nil {
return nil, err
}
}
if err := netlink.LinkSetUp(br); err != nil {
return nil, err
}
return br, nil
}
func setupVeth(netns ns.NetNS, br *netlink.Bridge, ifName string, mtu int, hairpinMode bool) error {
var hostVethName string
err := netns.Do(func(hostNS ns.NetNS) error {
// create the veth pair in the container and move host end into host netns
hostVeth, _, err := ip.SetupVeth(ifName, mtu, hostNS)
if err != nil {
return err
}
hostVethName = hostVeth.Attrs().Name
return nil
})
if err != nil {
return err
}
// need to lookup hostVeth again as its index has changed during ns move
hostVeth, err := netlink.LinkByName(hostVethName)
if err != nil {
return fmt.Errorf("failed to lookup %q: %v", hostVethName, err)
}
// connect host veth end to the bridge
if err = netlink.LinkSetMaster(hostVeth, br); err != nil {
return fmt.Errorf("failed to connect %q to bridge %v: %v", hostVethName, br.Attrs().Name, err)
}
// set hairpin mode
if err = netlink.LinkSetHairpin(hostVeth, hairpinMode); err != nil {
return fmt.Errorf("failed to setup hairpin mode for %v: %v", hostVethName, err)
}
return nil
}
func calcGatewayIP(ipn *net.IPNet) net.IP {
nid := ipn.IP.Mask(ipn.Mask)
return ip.NextIP(nid)
}
func setupBridge(n *NetConf) (*netlink.Bridge, error) {
// create bridge if necessary
br, err := ensureBridge(n.BrName, n.MTU)
if err != nil {
return nil, fmt.Errorf("failed to create bridge %q: %v", n.BrName, err)
}
return br, nil
}
func cmdAdd(args *skel.CmdArgs) error {
n, err := loadNetConf(args.StdinData)
if err != nil {
return err
}
if n.IsDefaultGW {
n.IsGW = true
}
br, err := setupBridge(n)
if err != nil {
return err
}
netns, err := ns.GetNS(args.Netns)
if err != nil {
return fmt.Errorf("failed to open netns %q: %v", args.Netns, err)
}
defer netns.Close()
if err = setupVeth(netns, br, args.IfName, n.MTU, n.HairpinMode); err != nil {
return err
}
// run the IPAM plugin and get back the config to apply
result, err := ipam.ExecAdd(n.IPAM.Type, args.StdinData)
if err != nil {
return err
}
// TODO: make this optional when IPv6 is supported
if result.IP4 == nil {
return errors.New("IPAM plugin returned missing IPv4 config")
}
if result.IP4.Gateway == nil && n.IsGW {
result.IP4.Gateway = calcGatewayIP(&result.IP4.IP)
}
if err := netns.Do(func(_ ns.NetNS) error {
// set the default gateway if requested
if n.IsDefaultGW {
_, defaultNet, err := net.ParseCIDR("0.0.0.0/0")
if err != nil {
return err
}
for _, route := range result.IP4.Routes {
if defaultNet.String() == route.Dst.String() {
if route.GW != nil && !route.GW.Equal(result.IP4.Gateway) {
return fmt.Errorf(
"isDefaultGateway ineffective because IPAM sets default route via %q",
route.GW,
)
}
}
}
result.IP4.Routes = append(
result.IP4.Routes,
types.Route{Dst: *defaultNet, GW: result.IP4.Gateway},
)
// TODO: IPV6
}
return ipam.ConfigureIface(args.IfName, result)
}); err != nil {
return err
}
if n.IsGW {
gwn := &net.IPNet{
IP: result.IP4.Gateway,
Mask: result.IP4.IP.Mask,
}
if err = ensureBridgeAddr(br, gwn); err != nil {
return err
}
if err := ip.EnableIP4Forward(); err != nil {
return fmt.Errorf("failed to enable forwarding: %v", err)
}
}
if n.IPMasq {
chain := utils.FormatChainName(n.Name, args.ContainerID)
comment := utils.FormatComment(n.Name, args.ContainerID)
if err = ip.SetupIPMasq(ip.Network(&result.IP4.IP), chain, comment); err != nil {
return err
}
}
result.DNS = n.DNS
return result.Print()
}
func cmdDel(args *skel.CmdArgs) error {
n, err := loadNetConf(args.StdinData)
if err != nil {
return err
}
if err := ipam.ExecDel(n.IPAM.Type, args.StdinData); err != nil {
return err
}
if args.Netns == "" {
return nil
}
var ipn *net.IPNet
err = ns.WithNetNSPath(args.Netns, func(_ ns.NetNS) error {
var err error
ipn, err = ip.DelLinkByNameAddr(args.IfName, netlink.FAMILY_V4)
return err
})
if err != nil {
return err
}
if n.IPMasq {
chain := utils.FormatChainName(n.Name, args.ContainerID)
comment := utils.FormatComment(n.Name, args.ContainerID)
if err = ip.TeardownIPMasq(ipn, chain, comment); err != nil {
return err
}
}
return nil
}
func Main() {
skel.PluginMain(cmdAdd, cmdDel)
}