Merge pull request #4312 from fidencio/topic/pass-the-tuntap-fd-to-clh

Allow Cloud Hypervisor to run under the `container_kvm_t`

src/runtime/virtcontainers/clh.go

@@ -9,11 +9,15 @@
 package virtcontainers
 
 import (
+	"bufio"
+	"bytes"
 	"context"
 	"encoding/json"
 	"fmt"
+	"io/ioutil"
 	"net"
 	"net/http"
+	"net/http/httputil"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -142,6 +146,80 @@ func (c *clhClientApi) VmRemoveDevicePut(ctx context.Context, vmRemoveDevice chc
 	return c.ApiInternal.VmRemoveDevicePut(ctx).VmRemoveDevice(vmRemoveDevice).Execute()
 }
 
+// This is done in order to be able to override such a function as part of
+// our unit tests, as when testing bootVM we're on a mocked scenario already.
+var vmAddNetPutRequest = func(clh *cloudHypervisor) error {
+	addr, err := net.ResolveUnixAddr("unix", clh.state.apiSocket)
+	if err != nil {
+		return err
+	}
+
+	conn, err := net.DialUnix("unix", nil, addr)
+	if err != nil {
+		return err
+	}
+	defer conn.Close()
+
+	for _, netDevice := range *clh.netDevices {
+		clh.Logger().Infof("Adding the net device to the Cloud Hypervisor VM configuration: %+v", netDevice)
+
+		netDeviceAsJson, err := json.Marshal(netDevice)
+		if err != nil {
+			return err
+		}
+		netDeviceAsIoReader := bytes.NewBuffer(netDeviceAsJson)
+
+		req, err := http.NewRequest(http.MethodPut, "http://localhost/api/v1/vm.add-net", netDeviceAsIoReader)
+		if err != nil {
+			return err
+		}
+
+		req.Header.Set("Accept", "application/json")
+		req.Header.Set("Content-Type", "application/json")
+		req.Header.Set("Content-Length", strconv.Itoa(int(netDeviceAsIoReader.Len())))
+
+		payload, err := httputil.DumpRequest(req, true)
+		if err != nil {
+			return err
+		}
+
+		files := clh.netDevicesFiles[*netDevice.Mac]
+		var fds []int
+		for _, f := range files {
+			fds = append(fds, int(f.Fd()))
+		}
+		oob := syscall.UnixRights(fds...)
+		payloadn, oobn, err := conn.WriteMsgUnix([]byte(payload), oob, nil)
+		if err != nil {
+			return err
+		}
+		if payloadn != len(payload) || oobn != len(oob) {
+			return fmt.Errorf("Failed to send all the request to Cloud Hypervisor. %d bytes expect to send as payload, %d bytes expect to send as oob date,  but only %d sent as payload, and %d sent as oob", len(payload), len(oob), payloadn, oobn)
+		}
+
+		reader := bufio.NewReader(conn)
+		resp, err := http.ReadResponse(reader, req)
+		if err != nil {
+			return err
+		}
+
+		respBody, err := ioutil.ReadAll(resp.Body)
+		if err != nil {
+			return err
+		}
+
+		resp.Body.Close()
+		resp.Body = ioutil.NopCloser(bytes.NewBuffer(respBody))
+
+		if resp.StatusCode != 204 {
+			clh.Logger().Errorf("vmAddNetPut failed with error '%d'. Response: %+v", resp.StatusCode, resp)
+			return fmt.Errorf("Failed to add the network device '%+v' to Cloud Hypervisor: %v", netDevice, resp.StatusCode)
+		}
+	}
+
+	return nil
+}
+
 //
 // Cloud hypervisor state
 //
@@ -159,15 +237,17 @@ func (s *CloudHypervisorState) reset() {
 }
 
 type cloudHypervisor struct {
-	console        console.Console
+	console         console.Console
-	virtiofsDaemon VirtiofsDaemon
+	virtiofsDaemon  VirtiofsDaemon
-	APIClient      clhClient
+	APIClient       clhClient
-	ctx            context.Context
+	ctx             context.Context
-	id             string
+	id              string
-	devicesIds     map[string]string
+	netDevices      *[]chclient.NetConfig
-	vmconfig       chclient.VmConfig
+	devicesIds      map[string]string
-	state          CloudHypervisorState
+	netDevicesFiles map[string][]*os.File
-	config         HypervisorConfig
+	vmconfig        chclient.VmConfig
+	state           CloudHypervisorState
+	config          HypervisorConfig
 }
 
 var clhKernelParams = []Param{
@@ -359,6 +439,7 @@ func (clh *cloudHypervisor) CreateVM(ctx context.Context, id string, network Net
 	clh.id = id
 	clh.state.state = clhNotReady
 	clh.devicesIds = make(map[string]string)
+	clh.netDevicesFiles = make(map[string][]*os.File)
 
 	clh.Logger().WithField("function", "CreateVM").Info("creating Sandbox")
 
@@ -1261,6 +1342,10 @@ func openAPIClientError(err error) error {
 	return fmt.Errorf("error: %v reason: %s", err, reason)
 }
 
+func (clh *cloudHypervisor) vmAddNetPut() error {
+	return vmAddNetPutRequest(clh)
+}
+
 func (clh *cloudHypervisor) bootVM(ctx context.Context) error {
 
 	cl := clh.client()
@@ -1288,6 +1373,11 @@ func (clh *cloudHypervisor) bootVM(ctx context.Context) error {
 		return fmt.Errorf("VM state is not 'Created' after 'CreateVM'")
 	}
 
+	err = clh.vmAddNetPut()
+	if err != nil {
+		return err
+	}
+
 	clh.Logger().Debug("Booting VM")
 	_, err = cl.BootVM(ctx)
 	if err != nil {
@@ -1369,34 +1459,30 @@ func (clh *cloudHypervisor) addNet(e Endpoint) error {
 	mac := e.HardwareAddr()
 	netPair := e.NetworkPair()
 	if netPair == nil {
-		return errors.New("net Pair to be added is nil, needed to get TAP path")
+		return errors.New("net Pair to be added is nil, needed to get TAP file descriptors")
 	}
 
-	tapPath := netPair.TapInterface.TAPIface.Name
+	if len(netPair.TapInterface.VMFds) == 0 {
-	if tapPath == "" {
+		return errors.New("The file descriptors for the network pair are not present")
-		return errors.New("TAP path in network pair is empty")
 	}
-
+	clh.netDevicesFiles[mac] = netPair.TapInterface.VMFds
-	clh.Logger().WithFields(log.Fields{
-		"mac": mac,
-		"tap": tapPath,
-	}).Info("Adding Net")
 
 	netRateLimiterConfig := clh.getNetRateLimiterConfig()
 
 	net := chclient.NewNetConfig()
 	net.Mac = &mac
-	net.Tap = &tapPath
 	if netRateLimiterConfig != nil {
 		net.SetRateLimiterConfig(*netRateLimiterConfig)
 	}
 
-	if clh.vmconfig.Net != nil {
+	if clh.netDevices != nil {
-		*clh.vmconfig.Net = append(*clh.vmconfig.Net, *net)
+		*clh.netDevices = append(*clh.netDevices, *net)
 	} else {
-		clh.vmconfig.Net = &[]chclient.NetConfig{*net}
+		clh.netDevices = &[]chclient.NetConfig{*net}
 	}
 
+	clh.Logger().Infof("Storing the Cloud Hypervisor network configuration: %+v", net)
+
 	return nil
 }
 

src/runtime/virtcontainers/clh_test.go

@@ -10,6 +10,7 @@ package virtcontainers
 
 import (
 	"context"
+	"io/ioutil"
 	"net/http"
 	"os"
 	"path/filepath"
@@ -129,33 +130,38 @@ func TestCloudHypervisorAddVSock(t *testing.T) {
 // Check addNet appends to the network config list new configurations.
 // Check that the elements in the list has the correct values
 func TestCloudHypervisorAddNetCheckNetConfigListValues(t *testing.T) {
-	macTest := "00:00:00:00:00"
-	tapPath := "/path/to/tap"
-
 	assert := assert.New(t)
 
+	macTest := "00:00:00:00:00"
+
+	file, err := ioutil.TempFile("", "netFd")
+	assert.Nil(err)
+	defer os.Remove(file.Name())
+
+	vmFds := make([]*os.File, 1)
+	vmFds = append(vmFds, file)
+
 	clh := cloudHypervisor{}
+	clh.netDevicesFiles = make(map[string][]*os.File)
 
 	e := &VethEndpoint{}
 	e.NetPair.TAPIface.HardAddr = macTest
-	e.NetPair.TapInterface.TAPIface.Name = tapPath
+	e.NetPair.TapInterface.VMFds = vmFds
 
-	err := clh.addNet(e)
+	err = clh.addNet(e)
 	assert.Nil(err)
 
-	assert.Equal(len(*clh.vmconfig.Net), 1)
+	assert.Equal(len(*clh.netDevices), 1)
 	if err == nil {
-		assert.Equal(*(*clh.vmconfig.Net)[0].Mac, macTest)
+		assert.Equal(*(*clh.netDevices)[0].Mac, macTest)
-		assert.Equal(*(*clh.vmconfig.Net)[0].Tap, tapPath)
 	}
 
 	err = clh.addNet(e)
 	assert.Nil(err)
 
-	assert.Equal(len(*clh.vmconfig.Net), 2)
+	assert.Equal(len(*clh.netDevices), 2)
 	if err == nil {
-		assert.Equal(*(*clh.vmconfig.Net)[1].Mac, macTest)
+		assert.Equal(*(*clh.netDevices)[1].Mac, macTest)
-		assert.Equal(*(*clh.vmconfig.Net)[1].Tap, tapPath)
 	}
 }
 
@@ -164,10 +170,18 @@ func TestCloudHypervisorAddNetCheckNetConfigListValues(t *testing.T) {
 func TestCloudHypervisorAddNetCheckEnpointTypes(t *testing.T) {
 	assert := assert.New(t)
 
-	tapPath := "/path/to/tap"
+	macTest := "00:00:00:00:00"
+
+	file, err := ioutil.TempFile("", "netFd")
+	assert.Nil(err)
+	defer os.Remove(file.Name())
+
+	vmFds := make([]*os.File, 1)
+	vmFds = append(vmFds, file)
 
 	validVeth := &VethEndpoint{}
-	validVeth.NetPair.TapInterface.TAPIface.Name = tapPath
+	validVeth.NetPair.TAPIface.HardAddr = macTest
+	validVeth.NetPair.TapInterface.VMFds = vmFds
 
 	type args struct {
 		e Endpoint
@@ -185,11 +199,12 @@ func TestCloudHypervisorAddNetCheckEnpointTypes(t *testing.T) {
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			clh := &cloudHypervisor{}
+			clh.netDevicesFiles = make(map[string][]*os.File)
 			if err := clh.addNet(tt.args.e); (err != nil) != tt.wantErr {
 				t.Errorf("cloudHypervisor.addNet() error = %v, wantErr %v", err, tt.wantErr)
-
 			} else if err == nil {
-				assert.Equal(*(*clh.vmconfig.Net)[0].Tap, tapPath)
+				files := clh.netDevicesFiles[macTest]
+				assert.Equal(files, vmFds)
 			}
 		})
 	}
@@ -199,10 +214,15 @@ func TestCloudHypervisorAddNetCheckEnpointTypes(t *testing.T) {
 func TestCloudHypervisorNetRateLimiter(t *testing.T) {
 	assert := assert.New(t)
 
-	tapPath := "/path/to/tap"
+	file, err := ioutil.TempFile("", "netFd")
+	assert.Nil(err)
+	defer os.Remove(file.Name())
+
+	vmFds := make([]*os.File, 1)
+	vmFds = append(vmFds, file)
 
 	validVeth := &VethEndpoint{}
-	validVeth.NetPair.TapInterface.TAPIface.Name = tapPath
+	validVeth.NetPair.TapInterface.VMFds = vmFds
 
 	type args struct {
 		bwMaxRate       int64
@@ -339,13 +359,14 @@ func TestCloudHypervisorNetRateLimiter(t *testing.T) {
 			clhConfig.NetRateLimiterOpsOneTimeBurst = tt.args.opsOneTimeBurst
 
 			clh := &cloudHypervisor{}
+			clh.netDevicesFiles = make(map[string][]*os.File)
 			clh.config = clhConfig
 			clh.APIClient = &clhClientMock{}
 
 			if err := clh.addNet(validVeth); err != nil {
 				t.Errorf("cloudHypervisor.addNet() error = %v", err)
 			} else {
-				netConfig := (*clh.vmconfig.Net)[0]
+				netConfig := (*clh.netDevices)[0]
 
 				assert.Equal(netConfig.HasRateLimiterConfig(), tt.expectsRateLimiter)
 				if tt.expectsRateLimiter {
@@ -373,6 +394,13 @@ func TestCloudHypervisorNetRateLimiter(t *testing.T) {
 func TestCloudHypervisorBootVM(t *testing.T) {
 	clh := &cloudHypervisor{}
 	clh.APIClient = &clhClientMock{}
+
+	savedVmAddNetPutRequestFunc := vmAddNetPutRequest
+	vmAddNetPutRequest = func(clh *cloudHypervisor) error { return nil }
+	defer func() {
+		vmAddNetPutRequest = savedVmAddNetPutRequestFunc
+	}()
+
 	var ctx context.Context
 	if err := clh.bootVM(ctx); err != nil {
 		t.Errorf("cloudHypervisor.bootVM() error = %v", err)
@@ -486,6 +514,12 @@ func TestCloudHypervisorStartSandbox(t *testing.T) {
 	store, err := persist.GetDriver()
 	assert.NoError(err)
 
+	savedVmAddNetPutRequestFunc := vmAddNetPutRequest
+	vmAddNetPutRequest = func(clh *cloudHypervisor) error { return nil }
+	defer func() {
+		vmAddNetPutRequest = savedVmAddNetPutRequestFunc
+	}()
+
 	clhConfig.VMStorePath = store.RunVMStoragePath()
 	clhConfig.RunStorePath = store.RunStoragePath()
 

src/runtime/virtcontainers/fc.go

@@ -927,6 +927,12 @@ func (fc *firecracker) fcAddNetDevice(ctx context.Context, endpoint Endpoint) {
 
 	ifaceID := endpoint.Name()
 
+	// VMFds are not used by Firecracker, as it opens the tuntap
+	// device by its name.  Let's just close those.
+	for _, f := range endpoint.NetworkPair().TapInterface.VMFds {
+		f.Close()
+	}
+
 	// The implementation of rate limiter is based on TBF.
 	// Rate Limiter defines a token bucket with a maximum capacity (size) to store tokens, and an interval for refilling purposes (refill_time).
 	// The refill-rate is derived from size and refill_time, and it is the constant rate at which the tokens replenish.

src/runtime/virtcontainers/network_linux.go

@@ -408,9 +408,19 @@ func createLink(netHandle *netlink.Handle, name string, expectedLink netlink.Lin
 
 	switch expectedLink.Type() {
 	case (&netlink.Tuntap{}).Type():
-		flags := netlink.TUNTAP_VNET_HDR
+		flags := netlink.TUNTAP_VNET_HDR | netlink.TUNTAP_NO_PI
 		if queues > 0 {
 			flags |= netlink.TUNTAP_MULTI_QUEUE_DEFAULTS
+		} else {
+			// We need to enforce `queues = 1` here in case
+			// multi-queue is *not* supported, the reason being
+			// `linkModify()`, a method called by `LinkAdd()`, only
+			// returning the file descriptor of the opened tuntap
+			// device when the queues are set to *non zero*.
+			//
+			// Please, for more information, refer to:
+			// https://github.com/kata-containers/kata-containers/blob/e6e5d2593ac319329269d7b58c30f99ba7b2bf5a/src/runtime/vendor/github.com/vishvananda/netlink/link_linux.go#L1164-L1316
+			queues = 1
 		}
 		newLink = &netlink.Tuntap{
 			LinkAttrs: netlink.LinkAttrs{Name: name},