github/kata-containers
1
0
Fork 1
mirror of https://github.com/kata-containers/kata-containers.git synced 2025-07-06 03:56:34 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #4312 from fidencio/topic/pass-the-tuntap-fd-to-clh

Browse Source 
Allow Cloud Hypervisor to run under the `container_kvm_t`
This commit is contained in:
James O. D. Hunt 2022-06-15 09:37:49 +01:00 committed by GitHub
commit d06dd8fcdc
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 177 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

130
src/runtime/virtcontainers/clh.go
View File

@ -9,11 +9,15 @@
package virtcontainers
import (
"bufio"
"bytes"
"context"
"encoding/json"
"fmt"
"io/ioutil"
"net"
"net/http"
"net/http/httputil"
"os"
"os/exec"
"path/filepath"
@ -142,6 +146,80 @@ func (c *clhClientApi) VmRemoveDevicePut(ctx context.Context, vmRemoveDevice chc
return c.ApiInternal.VmRemoveDevicePut(ctx).VmRemoveDevice(vmRemoveDevice).Execute()
}
// This is done in order to be able to override such a function as part of
// our unit tests, as when testing bootVM we're on a mocked scenario already.
var vmAddNetPutRequest = func(clh *cloudHypervisor) error {
addr, err := net.ResolveUnixAddr("unix", clh.state.apiSocket)
if err != nil {
return err
}
conn, err := net.DialUnix("unix", nil, addr)
if err != nil {
return err
}
defer conn.Close()
for _, netDevice := range *clh.netDevices {
clh.Logger().Infof("Adding the net device to the Cloud Hypervisor VM configuration: %+v", netDevice)
netDeviceAsJson, err := json.Marshal(netDevice)
if err != nil {
return err
}
netDeviceAsIoReader := bytes.NewBuffer(netDeviceAsJson)
req, err := http.NewRequest(http.MethodPut, "http://localhost/api/v1/vm.add-net", netDeviceAsIoReader)
if err != nil {
return err
}
req.Header.Set("Accept", "application/json")
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Content-Length", strconv.Itoa(int(netDeviceAsIoReader.Len())))
payload, err := httputil.DumpRequest(req, true)
if err != nil {
return err
}
files := clh.netDevicesFiles[*netDevice.Mac]
var fds []int
for _, f := range files {
fds = append(fds, int(f.Fd()))
}
oob := syscall.UnixRights(fds...)
payloadn, oobn, err := conn.WriteMsgUnix([]byte(payload), oob, nil)
if err != nil {
return err
}
if payloadn != len(payload) || oobn != len(oob) {
return fmt.Errorf("Failed to send all the request to Cloud Hypervisor. %d bytes expect to send as payload, %d bytes expect to send as oob date, but only %d sent as payload, and %d sent as oob", len(payload), len(oob), payloadn, oobn)
}
reader := bufio.NewReader(conn)
resp, err := http.ReadResponse(reader, req)
if err != nil {
return err
}
respBody, err := ioutil.ReadAll(resp.Body)
if err != nil {
return err
}
resp.Body.Close()
resp.Body = ioutil.NopCloser(bytes.NewBuffer(respBody))
if resp.StatusCode != 204 {
clh.Logger().Errorf("vmAddNetPut failed with error '%d'. Response: %+v", resp.StatusCode, resp)
return fmt.Errorf("Failed to add the network device '%+v' to Cloud Hypervisor: %v", netDevice, resp.StatusCode)
}
}
return nil
}
//
// Cloud hypervisor state
//
@ -159,15 +237,17 @@ func (s *CloudHypervisorState) reset() {
}
type cloudHypervisor struct {
console console.Console
virtiofsDaemon VirtiofsDaemon
APIClient clhClient
ctx context.Context
id string
devicesIds map[string]string
vmconfig chclient.VmConfig
state CloudHypervisorState
config HypervisorConfig
console console.Console
virtiofsDaemon VirtiofsDaemon
APIClient clhClient
ctx context.Context
id string
netDevices *[]chclient.NetConfig
devicesIds map[string]string
netDevicesFiles map[string][]*os.File
vmconfig chclient.VmConfig
state CloudHypervisorState
config HypervisorConfig
}
var clhKernelParams = []Param{
@ -359,6 +439,7 @@ func (clh *cloudHypervisor) CreateVM(ctx context.Context, id string, network Net
clh.id = id
clh.state.state = clhNotReady
clh.devicesIds = make(map[string]string)
clh.netDevicesFiles = make(map[string][]*os.File)
clh.Logger().WithField("function", "CreateVM").Info("creating Sandbox")
@ -1261,6 +1342,10 @@ func openAPIClientError(err error) error {
return fmt.Errorf("error: %v reason: %s", err, reason)
}
func (clh *cloudHypervisor) vmAddNetPut() error {
return vmAddNetPutRequest(clh)
}
func (clh *cloudHypervisor) bootVM(ctx context.Context) error {
cl := clh.client()
@ -1288,6 +1373,11 @@ func (clh *cloudHypervisor) bootVM(ctx context.Context) error {
return fmt.Errorf("VM state is not 'Created' after 'CreateVM'")
}
err = clh.vmAddNetPut()
if err != nil {
return err
}
clh.Logger().Debug("Booting VM")
_, err = cl.BootVM(ctx)
if err != nil {
@ -1369,34 +1459,30 @@ func (clh *cloudHypervisor) addNet(e Endpoint) error {
mac := e.HardwareAddr()
netPair := e.NetworkPair()
if netPair == nil {
return errors.New("net Pair to be added is nil, needed to get TAP path")
return errors.New("net Pair to be added is nil, needed to get TAP file descriptors")
}
tapPath := netPair.TapInterface.TAPIface.Name
if tapPath == "" {
return errors.New("TAP path in network pair is empty")
if len(netPair.TapInterface.VMFds) == 0 {
return errors.New("The file descriptors for the network pair are not present")
}
clh.Logger().WithFields(log.Fields{
"mac": mac,
"tap": tapPath,
}).Info("Adding Net")
clh.netDevicesFiles[mac] = netPair.TapInterface.VMFds
netRateLimiterConfig := clh.getNetRateLimiterConfig()
net := chclient.NewNetConfig()
net.Mac = &mac
net.Tap = &tapPath
if netRateLimiterConfig != nil {
net.SetRateLimiterConfig(*netRateLimiterConfig)
}
if clh.vmconfig.Net != nil {
*clh.vmconfig.Net = append(*clh.vmconfig.Net, *net)
if clh.netDevices != nil {
*clh.netDevices = append(*clh.netDevices, *net)
} else {
clh.vmconfig.Net = &[]chclient.NetConfig{*net}
clh.netDevices = &[]chclient.NetConfig{*net}
}
clh.Logger().Infof("Storing the Cloud Hypervisor network configuration: %+v", net)
return nil
}

70
src/runtime/virtcontainers/clh_test.go
View File

@ -10,6 +10,7 @@ package virtcontainers
import (
"context"
"io/ioutil"
"net/http"
"os"
"path/filepath"
@ -129,33 +130,38 @@ func TestCloudHypervisorAddVSock(t *testing.T) {
// Check addNet appends to the network config list new configurations.
// Check that the elements in the list has the correct values
func TestCloudHypervisorAddNetCheckNetConfigListValues(t *testing.T) {
macTest := "00:00:00:00:00"
tapPath := "/path/to/tap"
assert := assert.New(t)
macTest := "00:00:00:00:00"
file, err := ioutil.TempFile("", "netFd")
assert.Nil(err)
defer os.Remove(file.Name())
vmFds := make([]*os.File, 1)
vmFds = append(vmFds, file)
clh := cloudHypervisor{}
clh.netDevicesFiles = make(map[string][]*os.File)
e := &VethEndpoint{}
e.NetPair.TAPIface.HardAddr = macTest
e.NetPair.TapInterface.TAPIface.Name = tapPath
e.NetPair.TapInterface.VMFds = vmFds
err := clh.addNet(e)
err = clh.addNet(e)
assert.Nil(err)
assert.Equal(len(*clh.vmconfig.Net), 1)
assert.Equal(len(*clh.netDevices), 1)
if err == nil {
assert.Equal(*(*clh.vmconfig.Net)[0].Mac, macTest)
assert.Equal(*(*clh.vmconfig.Net)[0].Tap, tapPath)
assert.Equal(*(*clh.netDevices)[0].Mac, macTest)
}
err = clh.addNet(e)
assert.Nil(err)
assert.Equal(len(*clh.vmconfig.Net), 2)
assert.Equal(len(*clh.netDevices), 2)
if err == nil {
assert.Equal(*(*clh.vmconfig.Net)[1].Mac, macTest)
assert.Equal(*(*clh.vmconfig.Net)[1].Tap, tapPath)
assert.Equal(*(*clh.netDevices)[1].Mac, macTest)
}
}
@ -164,10 +170,18 @@ func TestCloudHypervisorAddNetCheckNetConfigListValues(t *testing.T) {
func TestCloudHypervisorAddNetCheckEnpointTypes(t *testing.T) {
assert := assert.New(t)
tapPath := "/path/to/tap"
macTest := "00:00:00:00:00"
file, err := ioutil.TempFile("", "netFd")
assert.Nil(err)
defer os.Remove(file.Name())
vmFds := make([]*os.File, 1)
vmFds = append(vmFds, file)
validVeth := &VethEndpoint{}
validVeth.NetPair.TapInterface.TAPIface.Name = tapPath
validVeth.NetPair.TAPIface.HardAddr = macTest
validVeth.NetPair.TapInterface.VMFds = vmFds
type args struct {
e Endpoint
@ -185,11 +199,12 @@ func TestCloudHypervisorAddNetCheckEnpointTypes(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
clh := &cloudHypervisor{}
clh.netDevicesFiles = make(map[string][]*os.File)
if err := clh.addNet(tt.args.e); (err != nil) != tt.wantErr {
t.Errorf("cloudHypervisor.addNet() error = %v, wantErr %v", err, tt.wantErr)
} else if err == nil {
assert.Equal(*(*clh.vmconfig.Net)[0].Tap, tapPath)
files := clh.netDevicesFiles[macTest]
assert.Equal(files, vmFds)
}
})
}
@ -199,10 +214,15 @@ func TestCloudHypervisorAddNetCheckEnpointTypes(t *testing.T) {
func TestCloudHypervisorNetRateLimiter(t *testing.T) {
assert := assert.New(t)
tapPath := "/path/to/tap"
file, err := ioutil.TempFile("", "netFd")
assert.Nil(err)
defer os.Remove(file.Name())
vmFds := make([]*os.File, 1)
vmFds = append(vmFds, file)
validVeth := &VethEndpoint{}
validVeth.NetPair.TapInterface.TAPIface.Name = tapPath
validVeth.NetPair.TapInterface.VMFds = vmFds
type args struct {
bwMaxRate int64
@ -339,13 +359,14 @@ func TestCloudHypervisorNetRateLimiter(t *testing.T) {
clhConfig.NetRateLimiterOpsOneTimeBurst = tt.args.opsOneTimeBurst
clh := &cloudHypervisor{}
clh.netDevicesFiles = make(map[string][]*os.File)
clh.config = clhConfig
clh.APIClient = &clhClientMock{}
if err := clh.addNet(validVeth); err != nil {
t.Errorf("cloudHypervisor.addNet() error = %v", err)
} else {
netConfig := (*clh.vmconfig.Net)[0]
netConfig := (*clh.netDevices)[0]
assert.Equal(netConfig.HasRateLimiterConfig(), tt.expectsRateLimiter)
if tt.expectsRateLimiter {
@ -373,6 +394,13 @@ func TestCloudHypervisorNetRateLimiter(t *testing.T) {
func TestCloudHypervisorBootVM(t *testing.T) {
clh := &cloudHypervisor{}
clh.APIClient = &clhClientMock{}
savedVmAddNetPutRequestFunc := vmAddNetPutRequest
vmAddNetPutRequest = func(clh *cloudHypervisor) error { return nil }
defer func() {
vmAddNetPutRequest = savedVmAddNetPutRequestFunc
}()
var ctx context.Context
if err := clh.bootVM(ctx); err != nil {
t.Errorf("cloudHypervisor.bootVM() error = %v", err)
@ -486,6 +514,12 @@ func TestCloudHypervisorStartSandbox(t *testing.T) {
store, err := persist.GetDriver()
assert.NoError(err)
savedVmAddNetPutRequestFunc := vmAddNetPutRequest
vmAddNetPutRequest = func(clh *cloudHypervisor) error { return nil }
defer func() {
vmAddNetPutRequest = savedVmAddNetPutRequestFunc
}()
clhConfig.VMStorePath = store.RunVMStoragePath()
clhConfig.RunStorePath = store.RunStoragePath()

6
src/runtime/virtcontainers/fc.go
View File

@ -927,6 +927,12 @@ func (fc *firecracker) fcAddNetDevice(ctx context.Context, endpoint Endpoint) {
ifaceID := endpoint.Name()
// VMFds are not used by Firecracker, as it opens the tuntap
// device by its name. Let's just close those.
for _, f := range endpoint.NetworkPair().TapInterface.VMFds {
f.Close()
}
// The implementation of rate limiter is based on TBF.
// Rate Limiter defines a token bucket with a maximum capacity (size) to store tokens, and an interval for refilling purposes (refill_time).
// The refill-rate is derived from size and refill_time, and it is the constant rate at which the tokens replenish.

12
src/runtime/virtcontainers/network_linux.go
View File

@ -408,9 +408,19 @@ func createLink(netHandle *netlink.Handle, name string, expectedLink netlink.Lin
switch expectedLink.Type() {
case (&netlink.Tuntap{}).Type():
flags := netlink.TUNTAP_VNET_HDR
flags := netlink.TUNTAP_VNET_HDR | netlink.TUNTAP_NO_PI
if queues > 0 {
flags |= netlink.TUNTAP_MULTI_QUEUE_DEFAULTS
} else {
// We need to enforce `queues = 1` here in case
// multi-queue is *not* supported, the reason being
// `linkModify()`, a method called by `LinkAdd()`, only
// returning the file descriptor of the opened tuntap
// device when the queues are set to *non zero*.
//
// Please, for more information, refer to:
// https://github.com/kata-containers/kata-containers/blob/e6e5d2593ac319329269d7b58c30f99ba7b2bf5a/src/runtime/vendor/github.com/vishvananda/netlink/link_linux.go#L1164-L1316
queues = 1
}
newLink = &netlink.Tuntap{
LinkAttrs: netlink.LinkAttrs{Name: name},