diff --git a/src/runtime/virtcontainers/clh.go b/src/runtime/virtcontainers/clh.go index a47c688f4..ce8f8fa57 100644 --- a/src/runtime/virtcontainers/clh.go +++ b/src/runtime/virtcontainers/clh.go @@ -9,11 +9,15 @@ package virtcontainers import ( + "bufio" + "bytes" "context" "encoding/json" "fmt" + "io/ioutil" "net" "net/http" + "net/http/httputil" "os" "os/exec" "path/filepath" @@ -142,6 +146,80 @@ func (c *clhClientApi) VmRemoveDevicePut(ctx context.Context, vmRemoveDevice chc return c.ApiInternal.VmRemoveDevicePut(ctx).VmRemoveDevice(vmRemoveDevice).Execute() } +// This is done in order to be able to override such a function as part of +// our unit tests, as when testing bootVM we're on a mocked scenario already. +var vmAddNetPutRequest = func(clh *cloudHypervisor) error { + addr, err := net.ResolveUnixAddr("unix", clh.state.apiSocket) + if err != nil { + return err + } + + conn, err := net.DialUnix("unix", nil, addr) + if err != nil { + return err + } + defer conn.Close() + + for _, netDevice := range *clh.netDevices { + clh.Logger().Infof("Adding the net device to the Cloud Hypervisor VM configuration: %+v", netDevice) + + netDeviceAsJson, err := json.Marshal(netDevice) + if err != nil { + return err + } + netDeviceAsIoReader := bytes.NewBuffer(netDeviceAsJson) + + req, err := http.NewRequest(http.MethodPut, "http://localhost/api/v1/vm.add-net", netDeviceAsIoReader) + if err != nil { + return err + } + + req.Header.Set("Accept", "application/json") + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Content-Length", strconv.Itoa(int(netDeviceAsIoReader.Len()))) + + payload, err := httputil.DumpRequest(req, true) + if err != nil { + return err + } + + files := clh.netDevicesFiles[*netDevice.Mac] + var fds []int + for _, f := range files { + fds = append(fds, int(f.Fd())) + } + oob := syscall.UnixRights(fds...) + payloadn, oobn, err := conn.WriteMsgUnix([]byte(payload), oob, nil) + if err != nil { + return err + } + if payloadn != len(payload) || oobn != len(oob) { + return fmt.Errorf("Failed to send all the request to Cloud Hypervisor. %d bytes expect to send as payload, %d bytes expect to send as oob date, but only %d sent as payload, and %d sent as oob", len(payload), len(oob), payloadn, oobn) + } + + reader := bufio.NewReader(conn) + resp, err := http.ReadResponse(reader, req) + if err != nil { + return err + } + + respBody, err := ioutil.ReadAll(resp.Body) + if err != nil { + return err + } + + resp.Body.Close() + resp.Body = ioutil.NopCloser(bytes.NewBuffer(respBody)) + + if resp.StatusCode != 204 { + clh.Logger().Errorf("vmAddNetPut failed with error '%d'. Response: %+v", resp.StatusCode, resp) + return fmt.Errorf("Failed to add the network device '%+v' to Cloud Hypervisor: %v", netDevice, resp.StatusCode) + } + } + + return nil +} + // // Cloud hypervisor state // @@ -159,15 +237,17 @@ func (s *CloudHypervisorState) reset() { } type cloudHypervisor struct { - console console.Console - virtiofsDaemon VirtiofsDaemon - APIClient clhClient - ctx context.Context - id string - devicesIds map[string]string - vmconfig chclient.VmConfig - state CloudHypervisorState - config HypervisorConfig + console console.Console + virtiofsDaemon VirtiofsDaemon + APIClient clhClient + ctx context.Context + id string + netDevices *[]chclient.NetConfig + devicesIds map[string]string + netDevicesFiles map[string][]*os.File + vmconfig chclient.VmConfig + state CloudHypervisorState + config HypervisorConfig } var clhKernelParams = []Param{ @@ -359,6 +439,7 @@ func (clh *cloudHypervisor) CreateVM(ctx context.Context, id string, network Net clh.id = id clh.state.state = clhNotReady clh.devicesIds = make(map[string]string) + clh.netDevicesFiles = make(map[string][]*os.File) clh.Logger().WithField("function", "CreateVM").Info("creating Sandbox") @@ -1261,6 +1342,10 @@ func openAPIClientError(err error) error { return fmt.Errorf("error: %v reason: %s", err, reason) } +func (clh *cloudHypervisor) vmAddNetPut() error { + return vmAddNetPutRequest(clh) +} + func (clh *cloudHypervisor) bootVM(ctx context.Context) error { cl := clh.client() @@ -1288,6 +1373,11 @@ func (clh *cloudHypervisor) bootVM(ctx context.Context) error { return fmt.Errorf("VM state is not 'Created' after 'CreateVM'") } + err = clh.vmAddNetPut() + if err != nil { + return err + } + clh.Logger().Debug("Booting VM") _, err = cl.BootVM(ctx) if err != nil { @@ -1369,34 +1459,30 @@ func (clh *cloudHypervisor) addNet(e Endpoint) error { mac := e.HardwareAddr() netPair := e.NetworkPair() if netPair == nil { - return errors.New("net Pair to be added is nil, needed to get TAP path") + return errors.New("net Pair to be added is nil, needed to get TAP file descriptors") } - tapPath := netPair.TapInterface.TAPIface.Name - if tapPath == "" { - return errors.New("TAP path in network pair is empty") + if len(netPair.TapInterface.VMFds) == 0 { + return errors.New("The file descriptors for the network pair are not present") } - - clh.Logger().WithFields(log.Fields{ - "mac": mac, - "tap": tapPath, - }).Info("Adding Net") + clh.netDevicesFiles[mac] = netPair.TapInterface.VMFds netRateLimiterConfig := clh.getNetRateLimiterConfig() net := chclient.NewNetConfig() net.Mac = &mac - net.Tap = &tapPath if netRateLimiterConfig != nil { net.SetRateLimiterConfig(*netRateLimiterConfig) } - if clh.vmconfig.Net != nil { - *clh.vmconfig.Net = append(*clh.vmconfig.Net, *net) + if clh.netDevices != nil { + *clh.netDevices = append(*clh.netDevices, *net) } else { - clh.vmconfig.Net = &[]chclient.NetConfig{*net} + clh.netDevices = &[]chclient.NetConfig{*net} } + clh.Logger().Infof("Storing the Cloud Hypervisor network configuration: %+v", net) + return nil } diff --git a/src/runtime/virtcontainers/clh_test.go b/src/runtime/virtcontainers/clh_test.go index 79e35210b..302d945f0 100644 --- a/src/runtime/virtcontainers/clh_test.go +++ b/src/runtime/virtcontainers/clh_test.go @@ -10,6 +10,7 @@ package virtcontainers import ( "context" + "io/ioutil" "net/http" "os" "path/filepath" @@ -129,33 +130,38 @@ func TestCloudHypervisorAddVSock(t *testing.T) { // Check addNet appends to the network config list new configurations. // Check that the elements in the list has the correct values func TestCloudHypervisorAddNetCheckNetConfigListValues(t *testing.T) { - macTest := "00:00:00:00:00" - tapPath := "/path/to/tap" - assert := assert.New(t) + macTest := "00:00:00:00:00" + + file, err := ioutil.TempFile("", "netFd") + assert.Nil(err) + defer os.Remove(file.Name()) + + vmFds := make([]*os.File, 1) + vmFds = append(vmFds, file) + clh := cloudHypervisor{} + clh.netDevicesFiles = make(map[string][]*os.File) e := &VethEndpoint{} e.NetPair.TAPIface.HardAddr = macTest - e.NetPair.TapInterface.TAPIface.Name = tapPath + e.NetPair.TapInterface.VMFds = vmFds - err := clh.addNet(e) + err = clh.addNet(e) assert.Nil(err) - assert.Equal(len(*clh.vmconfig.Net), 1) + assert.Equal(len(*clh.netDevices), 1) if err == nil { - assert.Equal(*(*clh.vmconfig.Net)[0].Mac, macTest) - assert.Equal(*(*clh.vmconfig.Net)[0].Tap, tapPath) + assert.Equal(*(*clh.netDevices)[0].Mac, macTest) } err = clh.addNet(e) assert.Nil(err) - assert.Equal(len(*clh.vmconfig.Net), 2) + assert.Equal(len(*clh.netDevices), 2) if err == nil { - assert.Equal(*(*clh.vmconfig.Net)[1].Mac, macTest) - assert.Equal(*(*clh.vmconfig.Net)[1].Tap, tapPath) + assert.Equal(*(*clh.netDevices)[1].Mac, macTest) } } @@ -164,10 +170,18 @@ func TestCloudHypervisorAddNetCheckNetConfigListValues(t *testing.T) { func TestCloudHypervisorAddNetCheckEnpointTypes(t *testing.T) { assert := assert.New(t) - tapPath := "/path/to/tap" + macTest := "00:00:00:00:00" + + file, err := ioutil.TempFile("", "netFd") + assert.Nil(err) + defer os.Remove(file.Name()) + + vmFds := make([]*os.File, 1) + vmFds = append(vmFds, file) validVeth := &VethEndpoint{} - validVeth.NetPair.TapInterface.TAPIface.Name = tapPath + validVeth.NetPair.TAPIface.HardAddr = macTest + validVeth.NetPair.TapInterface.VMFds = vmFds type args struct { e Endpoint @@ -185,11 +199,12 @@ func TestCloudHypervisorAddNetCheckEnpointTypes(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { clh := &cloudHypervisor{} + clh.netDevicesFiles = make(map[string][]*os.File) if err := clh.addNet(tt.args.e); (err != nil) != tt.wantErr { t.Errorf("cloudHypervisor.addNet() error = %v, wantErr %v", err, tt.wantErr) - } else if err == nil { - assert.Equal(*(*clh.vmconfig.Net)[0].Tap, tapPath) + files := clh.netDevicesFiles[macTest] + assert.Equal(files, vmFds) } }) } @@ -199,10 +214,15 @@ func TestCloudHypervisorAddNetCheckEnpointTypes(t *testing.T) { func TestCloudHypervisorNetRateLimiter(t *testing.T) { assert := assert.New(t) - tapPath := "/path/to/tap" + file, err := ioutil.TempFile("", "netFd") + assert.Nil(err) + defer os.Remove(file.Name()) + + vmFds := make([]*os.File, 1) + vmFds = append(vmFds, file) validVeth := &VethEndpoint{} - validVeth.NetPair.TapInterface.TAPIface.Name = tapPath + validVeth.NetPair.TapInterface.VMFds = vmFds type args struct { bwMaxRate int64 @@ -339,13 +359,14 @@ func TestCloudHypervisorNetRateLimiter(t *testing.T) { clhConfig.NetRateLimiterOpsOneTimeBurst = tt.args.opsOneTimeBurst clh := &cloudHypervisor{} + clh.netDevicesFiles = make(map[string][]*os.File) clh.config = clhConfig clh.APIClient = &clhClientMock{} if err := clh.addNet(validVeth); err != nil { t.Errorf("cloudHypervisor.addNet() error = %v", err) } else { - netConfig := (*clh.vmconfig.Net)[0] + netConfig := (*clh.netDevices)[0] assert.Equal(netConfig.HasRateLimiterConfig(), tt.expectsRateLimiter) if tt.expectsRateLimiter { @@ -373,6 +394,13 @@ func TestCloudHypervisorNetRateLimiter(t *testing.T) { func TestCloudHypervisorBootVM(t *testing.T) { clh := &cloudHypervisor{} clh.APIClient = &clhClientMock{} + + savedVmAddNetPutRequestFunc := vmAddNetPutRequest + vmAddNetPutRequest = func(clh *cloudHypervisor) error { return nil } + defer func() { + vmAddNetPutRequest = savedVmAddNetPutRequestFunc + }() + var ctx context.Context if err := clh.bootVM(ctx); err != nil { t.Errorf("cloudHypervisor.bootVM() error = %v", err) @@ -486,6 +514,12 @@ func TestCloudHypervisorStartSandbox(t *testing.T) { store, err := persist.GetDriver() assert.NoError(err) + savedVmAddNetPutRequestFunc := vmAddNetPutRequest + vmAddNetPutRequest = func(clh *cloudHypervisor) error { return nil } + defer func() { + vmAddNetPutRequest = savedVmAddNetPutRequestFunc + }() + clhConfig.VMStorePath = store.RunVMStoragePath() clhConfig.RunStorePath = store.RunStoragePath() diff --git a/src/runtime/virtcontainers/fc.go b/src/runtime/virtcontainers/fc.go index 3a89c7fa4..b792c90ac 100644 --- a/src/runtime/virtcontainers/fc.go +++ b/src/runtime/virtcontainers/fc.go @@ -927,6 +927,12 @@ func (fc *firecracker) fcAddNetDevice(ctx context.Context, endpoint Endpoint) { ifaceID := endpoint.Name() + // VMFds are not used by Firecracker, as it opens the tuntap + // device by its name. Let's just close those. + for _, f := range endpoint.NetworkPair().TapInterface.VMFds { + f.Close() + } + // The implementation of rate limiter is based on TBF. // Rate Limiter defines a token bucket with a maximum capacity (size) to store tokens, and an interval for refilling purposes (refill_time). // The refill-rate is derived from size and refill_time, and it is the constant rate at which the tokens replenish. diff --git a/src/runtime/virtcontainers/network_linux.go b/src/runtime/virtcontainers/network_linux.go index c4f2380e5..f3356bb1c 100644 --- a/src/runtime/virtcontainers/network_linux.go +++ b/src/runtime/virtcontainers/network_linux.go @@ -408,9 +408,19 @@ func createLink(netHandle *netlink.Handle, name string, expectedLink netlink.Lin switch expectedLink.Type() { case (&netlink.Tuntap{}).Type(): - flags := netlink.TUNTAP_VNET_HDR + flags := netlink.TUNTAP_VNET_HDR | netlink.TUNTAP_NO_PI if queues > 0 { flags |= netlink.TUNTAP_MULTI_QUEUE_DEFAULTS + } else { + // We need to enforce `queues = 1` here in case + // multi-queue is *not* supported, the reason being + // `linkModify()`, a method called by `LinkAdd()`, only + // returning the file descriptor of the opened tuntap + // device when the queues are set to *non zero*. + // + // Please, for more information, refer to: + // https://github.com/kata-containers/kata-containers/blob/e6e5d2593ac319329269d7b58c30f99ba7b2bf5a/src/runtime/vendor/github.com/vishvananda/netlink/link_linux.go#L1164-L1316 + queues = 1 } newLink = &netlink.Tuntap{ LinkAttrs: netlink.LinkAttrs{Name: name},