persist: merge "network.json"

Merge "network.json" into "persist.json" so that new store can manage
network part.

Signed-off-by: Wei Zhang <weizhang555.zw@gmail.com>

virtcontainers/bridgedmacvlan_endpoint.go

@@ -9,6 +9,7 @@ import (
 	"fmt"
 
 	"github.com/containernetworking/plugins/pkg/ns"
+	persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api"
 )
 
 // BridgedMacvlanEndpoint represents a macvlan endpoint that is bridged to the VM
@@ -115,3 +116,52 @@ func (endpoint *BridgedMacvlanEndpoint) HotAttach(h hypervisor) error {
 func (endpoint *BridgedMacvlanEndpoint) HotDetach(h hypervisor, netNsCreated bool, netNsPath string) error {
 	return fmt.Errorf("BridgedMacvlanEndpoint does not support Hot detach")
 }
+
+func (endpoint *BridgedMacvlanEndpoint) save() (s persistapi.NetworkEndpoint) {
+	s.Type = string(endpoint.Type())
+	s.BridgedMacvlan = &persistapi.BridgedMacvlanEndpoint{
+		NetPair: persistapi.NetworkInterfacePair{
+			TapInterface: persistapi.TapInterface{
+				ID:   endpoint.NetPair.TapInterface.ID,
+				Name: endpoint.NetPair.TapInterface.Name,
+				TAPIface: persistapi.NetworkInterface{
+					Name:     endpoint.NetPair.TapInterface.TAPIface.Name,
+					HardAddr: endpoint.NetPair.TapInterface.TAPIface.HardAddr,
+					Addrs:    endpoint.NetPair.TapInterface.TAPIface.Addrs,
+				},
+			},
+			VirtIface: persistapi.NetworkInterface{
+				Name:     endpoint.NetPair.VirtIface.Name,
+				HardAddr: endpoint.NetPair.VirtIface.HardAddr,
+				Addrs:    endpoint.NetPair.VirtIface.Addrs,
+			},
+			NetInterworkingModel: int(endpoint.NetPair.NetInterworkingModel),
+		},
+	}
+	return
+}
+
+func (endpoint *BridgedMacvlanEndpoint) load(s persistapi.NetworkEndpoint) {
+	endpoint.EndpointType = BridgedMacvlanEndpointType
+
+	if s.BridgedMacvlan != nil {
+		iface := s.BridgedMacvlan
+		endpoint.NetPair = NetworkInterfacePair{
+			TapInterface: TapInterface{
+				ID:   iface.NetPair.TapInterface.ID,
+				Name: iface.NetPair.TapInterface.Name,
+				TAPIface: NetworkInterface{
+					Name:     iface.NetPair.TapInterface.TAPIface.Name,
+					HardAddr: iface.NetPair.TapInterface.TAPIface.HardAddr,
+					Addrs:    iface.NetPair.TapInterface.TAPIface.Addrs,
+				},
+			},
+			VirtIface: NetworkInterface{
+				Name:     iface.NetPair.VirtIface.Name,
+				HardAddr: iface.NetPair.VirtIface.HardAddr,
+				Addrs:    iface.NetPair.VirtIface.Addrs,
+			},
+			NetInterworkingModel: NetInterworkingModel(iface.NetPair.NetInterworkingModel),
+		}
+	}
+}

virtcontainers/endpoint.go

@@ -7,6 +7,8 @@ package virtcontainers
 
 import (
 	"fmt"
+
+	persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api"
 )
 
 // Endpoint represents a physical or virtual network interface.
@@ -24,6 +26,9 @@ type Endpoint interface {
 	Detach(netNsCreated bool, netNsPath string) error
 	HotAttach(h hypervisor) error
 	HotDetach(h hypervisor, netNsCreated bool, netNsPath string) error
+
+	save() persistapi.NetworkEndpoint
+	load(persistapi.NetworkEndpoint)
 }
 
 // EndpointType identifies the type of the network endpoint.

virtcontainers/fc.go

@@ -324,12 +324,14 @@ func (fc *firecracker) fcInit(timeout int) error {
 
 	// Fetch sandbox network to be able to access it from the sandbox structure.
 	var networkNS NetworkNamespace
+	if fc.store != nil {
 		if err := fc.store.Load(store.Network, &networkNS); err == nil {
 			if networkNS.NetNsPath == "" {
 				fc.Logger().WithField("NETWORK NAMESPACE NULL", networkNS).Warn()
 			}
 			fc.netNSPath = networkNS.NetNsPath
 		}
+	}
 
 	err := os.MkdirAll(fc.jailerRoot, store.DirMode)
 	if err != nil {

virtcontainers/ipvlan_endpoint.go

@@ -9,6 +9,7 @@ import (
 	"fmt"
 
 	"github.com/containernetworking/plugins/pkg/ns"
+	persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api"
 )
 
 // IPVlanEndpoint represents a ipvlan endpoint that is bridged to the VM
@@ -118,3 +119,52 @@ func (endpoint *IPVlanEndpoint) HotAttach(h hypervisor) error {
 func (endpoint *IPVlanEndpoint) HotDetach(h hypervisor, netNsCreated bool, netNsPath string) error {
 	return fmt.Errorf("IPVlanEndpoint does not support Hot detach")
 }
+
+func (endpoint *IPVlanEndpoint) save() (s persistapi.NetworkEndpoint) {
+	s.Type = string(endpoint.Type())
+	s.IPVlan = &persistapi.IPVlanEndpoint{
+		NetPair: persistapi.NetworkInterfacePair{
+			TapInterface: persistapi.TapInterface{
+				ID:   endpoint.NetPair.TapInterface.ID,
+				Name: endpoint.NetPair.TapInterface.Name,
+				TAPIface: persistapi.NetworkInterface{
+					Name:     endpoint.NetPair.TapInterface.TAPIface.Name,
+					HardAddr: endpoint.NetPair.TapInterface.TAPIface.HardAddr,
+					Addrs:    endpoint.NetPair.TapInterface.TAPIface.Addrs,
+				},
+			},
+			VirtIface: persistapi.NetworkInterface{
+				Name:     endpoint.NetPair.VirtIface.Name,
+				HardAddr: endpoint.NetPair.VirtIface.HardAddr,
+				Addrs:    endpoint.NetPair.VirtIface.Addrs,
+			},
+			NetInterworkingModel: int(endpoint.NetPair.NetInterworkingModel),
+		},
+	}
+	return
+}
+
+func (endpoint *IPVlanEndpoint) load(s persistapi.NetworkEndpoint) {
+	endpoint.EndpointType = IPVlanEndpointType
+
+	if s.IPVlan != nil {
+		iface := s.IPVlan
+		endpoint.NetPair = NetworkInterfacePair{
+			TapInterface: TapInterface{
+				ID:   iface.NetPair.TapInterface.ID,
+				Name: iface.NetPair.TapInterface.Name,
+				TAPIface: NetworkInterface{
+					Name:     iface.NetPair.TapInterface.TAPIface.Name,
+					HardAddr: iface.NetPair.TapInterface.TAPIface.HardAddr,
+					Addrs:    iface.NetPair.TapInterface.TAPIface.Addrs,
+				},
+			},
+			VirtIface: NetworkInterface{
+				Name:     iface.NetPair.VirtIface.Name,
+				HardAddr: iface.NetPair.VirtIface.HardAddr,
+				Addrs:    iface.NetPair.VirtIface.Addrs,
+			},
+			NetInterworkingModel: NetInterworkingModel(iface.NetPair.NetInterworkingModel),
+		}
+	}
+}

virtcontainers/macvtap_endpoint.go

@@ -8,6 +8,8 @@ package virtcontainers
 import (
 	"fmt"
 	"os"
+
+	persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api"
 )
 
 // MacvtapEndpoint represents a macvtap endpoint
@@ -102,3 +104,17 @@ func (endpoint *MacvtapEndpoint) SetPciAddr(pciAddr string) {
 func (endpoint *MacvtapEndpoint) NetworkPair() *NetworkInterfacePair {
 	return nil
 }
+
+func (endpoint *MacvtapEndpoint) save() (s persistapi.NetworkEndpoint) {
+	s.Type = string(endpoint.Type())
+	s.Macvtap = &persistapi.MacvtapEndpoint{
+		PCIAddr: endpoint.PCIAddr,
+	}
+	return
+}
+func (endpoint *MacvtapEndpoint) load(s persistapi.NetworkEndpoint) {
+	endpoint.EndpointType = MacvtapEndpointType
+	if s.Macvtap != nil {
+		endpoint.PCIAddr = s.Macvtap.PCIAddr
+	}
+}

virtcontainers/persist.go

@@ -160,6 +160,17 @@ func (s *Sandbox) dumpAgent(ss *persistapi.SandboxState) {
 	}
 }
 
+func (s *Sandbox) dumpNetwork(ss *persistapi.SandboxState) {
+	ss.Network = persistapi.NetworkInfo{
+		NetNsPath:    s.networkNS.NetNsPath,
+		NetmonPID:    s.networkNS.NetmonPID,
+		NetNsCreated: s.networkNS.NetNsCreated,
+	}
+	for _, e := range s.networkNS.Endpoints {
+		ss.Network.Endpoints = append(ss.Network.Endpoints, e.save())
+	}
+}
+
 func (s *Sandbox) Save() error {
 	var (
 		ss = persistapi.SandboxState{}
@@ -173,6 +184,7 @@ func (s *Sandbox) Save() error {
 	s.dumpProcess(cs)
 	s.dumpMounts(cs)
 	s.dumpAgent(&ss)
+	s.dumpNetwork(&ss)
 
 	if err := s.newStore.ToDisk(ss, cs); err != nil {
 		return err
@@ -248,6 +260,38 @@ func (c *Container) loadContProcess(cs persistapi.ContainerState) {
 	}
 }
 
+func (s *Sandbox) loadNetwork(netInfo persistapi.NetworkInfo) {
+	s.networkNS = NetworkNamespace{
+		NetNsPath:    netInfo.NetNsPath,
+		NetmonPID:    netInfo.NetmonPID,
+		NetNsCreated: netInfo.NetNsCreated,
+	}
+
+	for _, e := range netInfo.Endpoints {
+		var ep Endpoint
+		switch EndpointType(e.Type) {
+		case PhysicalEndpointType:
+			ep = &PhysicalEndpoint{}
+		case VethEndpointType:
+			ep = &VethEndpoint{}
+		case VhostUserEndpointType:
+			ep = &VhostUserEndpoint{}
+		case BridgedMacvlanEndpointType:
+			ep = &BridgedMacvlanEndpoint{}
+		case MacvtapEndpointType:
+			ep = &MacvtapEndpoint{}
+		case TapEndpointType:
+			ep = &TapEndpoint{}
+		case IPVlanEndpointType:
+			ep = &IPVlanEndpoint{}
+		default:
+			continue
+		}
+		ep.load(e)
+		s.networkNS.Endpoints = append(s.networkNS.Endpoints, ep)
+	}
+}
+
 // Restore will restore sandbox data from persist file on disk
 func (s *Sandbox) Restore() error {
 	ss, _, err := s.newStore.FromDisk(s.id)
@@ -259,6 +303,7 @@ func (s *Sandbox) Restore() error {
 	s.loadHypervisor(ss.HypervisorState)
 	s.loadDevices(ss.Devices)
 	s.loadAgent(ss.AgentState)
+	s.loadNetwork(ss.Network)
 	return nil
 }
 

virtcontainers/persist/api/network.go

@@ -6,20 +6,80 @@
 
 package persistapi
 
+import (
+	"github.com/vishvananda/netlink"
+)
+
 // ============= sandbox level resources =============
 
+type NetworkInterface struct {
+	Name     string
+	HardAddr string
+	Addrs    []netlink.Addr
+}
+
+// TapInterface defines a tap interface
+type TapInterface struct {
+	ID       string
+	Name     string
+	TAPIface NetworkInterface
+	// remove VMFds and VhostFds
+}
+
+// NetworkInterfacePair defines a pair between VM and virtual network interfaces.
+type NetworkInterfacePair struct {
+	TapInterface
+	VirtIface            NetworkInterface
+	NetInterworkingModel int
+}
+
+type PhysicalEndpoint struct {
+	BDF            string
+	Driver         string
+	VendorDeviceID string
+}
+
+type MacvtapEndpoint struct {
+	// This is for showing information.
+	// Remove this field won't impact anything.
+	PCIAddr string
+}
+
+type TapEndpoint struct {
+	TapInterface TapInterface
+}
+
+type BridgedMacvlanEndpoint struct {
+	NetPair NetworkInterfacePair
+}
+
+type VethEndpoint struct {
+	NetPair NetworkInterfacePair
+}
+
+type IPVlanEndpoint struct {
+	NetPair NetworkInterfacePair
+}
+
+type VhostUserEndpoint struct {
+	// This is for showing information.
+	// Remove these fields won't impact anything.
+	IfaceName string
+	PCIAddr   string
+}
+
 // NetworkEndpoint contains network interface information
 type NetworkEndpoint struct {
 	Type string
 
-	// ID used to pass the netdev option to qemu
+	// One and only one of these below are not nil according to Type.
-	ID string
+	Physical       *PhysicalEndpoint       `json:",omitempty"`
-
+	Veth           *VethEndpoint           `json:",omitempty"`
-	// Name of the interface
+	VhostUser      *VhostUserEndpoint      `json:",omitempty"`
-	Name string
+	BridgedMacvlan *BridgedMacvlanEndpoint `json:",omitempty"`
-
+	Macvtap        *MacvtapEndpoint        `json:",omitempty"`
-	// Index of interface
+	Tap            *TapEndpoint            `json:",omitempty"`
-	Index int
+	IPVlan         *IPVlanEndpoint         `json:",omitempty"`
 }
 
 // NetworkInfo contains network information of sandbox
@@ -27,6 +87,5 @@ type NetworkInfo struct {
 	NetNsPath    string
 	NetmonPID    int
 	NetNsCreated bool
-	InterworkingModel string
 	Endpoints    []NetworkEndpoint
 }

virtcontainers/physical_endpoint.go

@@ -14,6 +14,7 @@ import (
 
 	"github.com/kata-containers/runtime/virtcontainers/device/config"
 	"github.com/kata-containers/runtime/virtcontainers/device/drivers"
+	persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api"
 	"github.com/safchain/ethtool"
 )
 
@@ -200,3 +201,22 @@ func bindNICToVFIO(endpoint *PhysicalEndpoint) error {
 func bindNICToHost(endpoint *PhysicalEndpoint) error {
 	return drivers.BindDevicetoHost(endpoint.BDF, endpoint.Driver, endpoint.VendorDeviceID)
 }
+
+func (endpoint *PhysicalEndpoint) save() (s persistapi.NetworkEndpoint) {
+	s.Type = string(endpoint.Type())
+	s.Physical = &persistapi.PhysicalEndpoint{
+		BDF:            endpoint.BDF,
+		Driver:         endpoint.Driver,
+		VendorDeviceID: endpoint.VendorDeviceID,
+	}
+	return
+}
+
+func (endpoint *PhysicalEndpoint) load(s persistapi.NetworkEndpoint) {
+	endpoint.EndpointType = PhysicalEndpointType
+	if s.Physical != nil {
+		endpoint.BDF = s.Physical.BDF
+		endpoint.Driver = s.Physical.Driver
+		endpoint.VendorDeviceID = s.Physical.VendorDeviceID
+	}
+}

virtcontainers/sandbox.go

@@ -467,18 +467,18 @@ func createSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Fac
 		s.Logger().WithField("features", s.config.Experimental).Infof("Enable experimental features")
 	}
 
-	// Fetch sandbox network to be able to access it from the sandbox structure.
-	var networkNS NetworkNamespace
-	if err := s.store.Load(store.Network, &networkNS); err == nil {
-		s.networkNS = networkNS
-	}
-
 	if s.supportNewStore() {
 		// Restored successfully from newstore before.
 		if s.state.State != "" {
 			return s, nil
 		}
 	} else {
+		// Fetch sandbox network to be able to access it from the sandbox structure.
+		var networkNS NetworkNamespace
+		if err := s.store.Load(store.Network, &networkNS); err == nil {
+			s.networkNS = networkNS
+		}
+
 		devices, err := s.store.LoadDevices()
 		if err != nil {
 			s.Logger().WithError(err).WithField("sandboxid", s.id).Warning("load sandbox devices failed")
@@ -563,11 +563,6 @@ func newSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Factor
 		if err != nil {
 			s.Logger().WithError(err).WithField("sandboxid", s.id).Error("Create new sandbox failed")
 			globalSandboxList.removeSandbox(s.id)
-		}
-	}()
-
-	defer func() {
-		if err != nil {
 			s.store.Delete()
 		}
 	}()
@@ -846,8 +841,11 @@ func (s *Sandbox) createNetwork() error {
 	}
 
 	// Store the network
+	if !s.supportNewStore() {
 		return s.store.Store(store.Network, s.networkNS)
 	}
+	return nil
+}
 
 func (s *Sandbox) postCreatedNetwork() error {
 
@@ -919,14 +917,14 @@ func (s *Sandbox) AddInterface(inf *vcTypes.Interface) (*vcTypes.Interface, erro
 
 	// Update the sandbox storage
 	s.networkNS.Endpoints = append(s.networkNS.Endpoints, endpoint)
-	if err := s.store.Store(store.Network, s.networkNS); err != nil {
-		return nil, err
-	}
-
 	if s.supportNewStore() {
 		if err := s.Save(); err != nil {
 			return nil, err
 		}
+	} else {
+		if err := s.store.Store(store.Network, s.networkNS); err != nil {
+			return nil, err
+		}
 	}
 
 	// Add network for vm
@@ -943,14 +941,15 @@ func (s *Sandbox) RemoveInterface(inf *vcTypes.Interface) (*vcTypes.Interface, e
 				return inf, err
 			}
 			s.networkNS.Endpoints = append(s.networkNS.Endpoints[:i], s.networkNS.Endpoints[i+1:]...)
-			if err := s.store.Store(store.Network, s.networkNS); err != nil {
-				return inf, err
-			}
 
 			if s.supportNewStore() {
 				if err := s.Save(); err != nil {
 					return inf, err
 				}
+			} else {
+				if err := s.store.Store(store.Network, s.networkNS); err != nil {
+					return inf, err
+				}
 			}
 
 			break
@@ -1024,10 +1023,13 @@ func (s *Sandbox) startVM() (err error) {
 				return err
 			}
 		}
+
+		if !s.supportNewStore() {
 			if err := s.store.Store(store.Network, s.networkNS); err != nil {
 				return err
 			}
 		}
+	}
 
 	s.Logger().Info("VM started")
 

virtcontainers/tap_endpoint.go

@@ -11,6 +11,7 @@ import (
 	"github.com/containernetworking/plugins/pkg/ns"
 	"github.com/vishvananda/netlink"
 
+	persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api"
 	"github.com/kata-containers/runtime/virtcontainers/pkg/uuid"
 )
 
@@ -187,3 +188,35 @@ func unTapNetwork(name string) error {
 	}
 	return nil
 }
+
+func (endpoint *TapEndpoint) save() (s persistapi.NetworkEndpoint) {
+	s.Type = string(endpoint.Type())
+	s.Tap = &persistapi.TapEndpoint{
+		TapInterface: persistapi.TapInterface{
+			ID:   endpoint.TapInterface.ID,
+			Name: endpoint.TapInterface.Name,
+			TAPIface: persistapi.NetworkInterface{
+				Name:     endpoint.TapInterface.TAPIface.Name,
+				HardAddr: endpoint.TapInterface.TAPIface.HardAddr,
+				Addrs:    endpoint.TapInterface.TAPIface.Addrs,
+			},
+		},
+	}
+	return
+}
+func (endpoint *TapEndpoint) load(s persistapi.NetworkEndpoint) {
+	endpoint.EndpointType = TapEndpointType
+
+	if s.Tap != nil {
+		iface := s.Tap
+		endpoint.TapInterface = TapInterface{
+			ID:   iface.TapInterface.ID,
+			Name: iface.TapInterface.Name,
+			TAPIface: NetworkInterface{
+				Name:     iface.TapInterface.TAPIface.Name,
+				HardAddr: iface.TapInterface.TAPIface.HardAddr,
+				Addrs:    iface.TapInterface.TAPIface.Addrs,
+			},
+		}
+	}
+}

virtcontainers/veth_endpoint.go

@@ -9,6 +9,7 @@ import (
 	"fmt"
 
 	"github.com/containernetworking/plugins/pkg/ns"
+	persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api"
 )
 
 // VethEndpoint gathers a network pair and its properties.
@@ -141,3 +142,52 @@ func (endpoint *VethEndpoint) HotDetach(h hypervisor, netNsCreated bool, netNsPa
 	}
 	return nil
 }
+
+func (endpoint *VethEndpoint) save() (s persistapi.NetworkEndpoint) {
+	s.Type = string(endpoint.Type())
+	s.Veth = &persistapi.VethEndpoint{
+		NetPair: persistapi.NetworkInterfacePair{
+			TapInterface: persistapi.TapInterface{
+				ID:   endpoint.NetPair.TapInterface.ID,
+				Name: endpoint.NetPair.TapInterface.Name,
+				TAPIface: persistapi.NetworkInterface{
+					Name:     endpoint.NetPair.TapInterface.TAPIface.Name,
+					HardAddr: endpoint.NetPair.TapInterface.TAPIface.HardAddr,
+					Addrs:    endpoint.NetPair.TapInterface.TAPIface.Addrs,
+				},
+			},
+			VirtIface: persistapi.NetworkInterface{
+				Name:     endpoint.NetPair.VirtIface.Name,
+				HardAddr: endpoint.NetPair.VirtIface.HardAddr,
+				Addrs:    endpoint.NetPair.VirtIface.Addrs,
+			},
+			NetInterworkingModel: int(endpoint.NetPair.NetInterworkingModel),
+		},
+	}
+	return
+}
+
+func (endpoint *VethEndpoint) load(s persistapi.NetworkEndpoint) {
+	endpoint.EndpointType = VethEndpointType
+
+	if s.Veth != nil {
+		iface := s.Veth
+		endpoint.NetPair = NetworkInterfacePair{
+			TapInterface: TapInterface{
+				ID:   iface.NetPair.TapInterface.ID,
+				Name: iface.NetPair.TapInterface.Name,
+				TAPIface: NetworkInterface{
+					Name:     iface.NetPair.TapInterface.TAPIface.Name,
+					HardAddr: iface.NetPair.TapInterface.TAPIface.HardAddr,
+					Addrs:    iface.NetPair.TapInterface.TAPIface.Addrs,
+				},
+			},
+			VirtIface: NetworkInterface{
+				Name:     iface.NetPair.VirtIface.Name,
+				HardAddr: iface.NetPair.VirtIface.HardAddr,
+				Addrs:    iface.NetPair.VirtIface.Addrs,
+			},
+			NetInterworkingModel: NetInterworkingModel(iface.NetPair.NetInterworkingModel),
+		}
+	}
+}

virtcontainers/vhostuser_endpoint.go

@@ -11,6 +11,7 @@ import (
 	"os"
 
 	"github.com/kata-containers/runtime/virtcontainers/device/config"
+	persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api"
 	"github.com/kata-containers/runtime/virtcontainers/utils"
 )
 
@@ -149,3 +150,20 @@ func vhostUserSocketPath(info interface{}) (string, error) {
 	}
 
 }
+
+func (endpoint *VhostUserEndpoint) save() (s persistapi.NetworkEndpoint) {
+	s.Type = string(endpoint.Type())
+	s.VhostUser = &persistapi.VhostUserEndpoint{
+		IfaceName: endpoint.IfaceName,
+		PCIAddr:   endpoint.PCIAddr,
+	}
+	return
+}
+
+func (endpoint *VhostUserEndpoint) load(s persistapi.NetworkEndpoint) {
+	endpoint.EndpointType = VhostUserEndpointType
+	if s.VhostUser != nil {
+		endpoint.IfaceName = s.VhostUser.IfaceName
+		endpoint.PCIAddr = s.VhostUser.PCIAddr
+	}
+}