Merge pull request #3697 from yoheiueda/CCv0-peerpod

Support Peer Pod VMs

src/agent/src/rpc.rs

@@ -1989,6 +1989,38 @@ fn do_copy_file(req: &CopyFileRequest) -> Result<()> {
 
     std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(req.dir_mode))?;
 
+    let sflag = stat::SFlag::from_bits_truncate(req.file_mode);
+
+    if sflag.contains(stat::SFlag::S_IFDIR) {
+        fs::create_dir(path.clone()).or_else(|e| {
+            if e.kind() != std::io::ErrorKind::AlreadyExists {
+                return Err(e);
+            }
+            Ok(())
+        })?;
+
+        std::fs::set_permissions(path.clone(), std::fs::Permissions::from_mode(req.file_mode))?;
+
+        unistd::chown(
+            &path,
+            Some(Uid::from_raw(req.uid as u32)),
+            Some(Gid::from_raw(req.gid as u32)),
+        )?;
+
+        return Ok(());
+    }
+
+    if sflag.contains(stat::SFlag::S_IFLNK) {
+        let src = PathBuf::from(String::from_utf8(req.data.clone()).unwrap());
+
+        unistd::symlinkat(&src, None, &path)?;
+        let path_str = CString::new(path.to_str().unwrap())?;
+        let ret = unsafe { libc::lchown(path_str.as_ptr(), req.uid as u32, req.gid as u32) };
+        Errno::result(ret).map(drop)?;
+
+        return Ok(());
+    }
+
     let mut tmpfile = path.clone();
     tmpfile.set_extension("tmp");
 

src/runtime/cmd/kata-runtime/kata-check_amd64.go

@@ -304,6 +304,8 @@ func archHostCanCreateVMContainer(hypervisorType vc.HypervisorType) error {
 		return kvmIsUsable()
 	case "acrn":
 		return acrnIsUsable()
+	case "remote":
+		return nil
 	case "mock":
 		return nil
 	default:

src/runtime/cmd/kata-runtime/kata-check_arm64.go

@@ -86,6 +86,9 @@ func checkKVMExtensions() error {
 }
 
 func archHostCanCreateVMContainer(hypervisorType vc.HypervisorType) error {
+	if hypervisorType == "remote" {
+		return nil
+	}
 	if err := kvmIsUsable(); err != nil {
 		return err
 	}

src/runtime/cmd/kata-runtime/kata-check_ppc64le.go

@@ -61,6 +61,9 @@ func setCPUtype(hypervisorType vc.HypervisorType) error {
 }
 
 func archHostCanCreateVMContainer(hypervisorType vc.HypervisorType) error {
+	if hypervisorType == "remote" {
+		return nil
+	}
 	return kvmIsUsable()
 }
 

src/runtime/cmd/kata-runtime/kata-check_s390x.go

@@ -55,6 +55,9 @@ func kvmIsUsable() error {
 }
 
 func archHostCanCreateVMContainer(hypervisorType vc.HypervisorType) error {
+	if hypervisorType == "remote" {
+		return nil
+	}
 	return kvmIsUsable()
 }
 

src/runtime/hack/update-generated-hypervisor-proto.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+# (C) Copyright IBM Corp. 2022.
+# SPDX-License-Identifier: Apache-2.0
+
+set -o errexit -o pipefail -o nounset
+
+cd "$(dirname "${BASH_SOURCE[0]}")/.."
+
+protoc --gogottrpc_out=protocols/hypervisor \
+	--gogottrpc_opt=plugins=ttrpc+fieldpath,paths=source_relative \
+	-Iprotocols/hypervisor \
+	-I../libs/protocols/protos/gogo/protobuf \
+	protocols/hypervisor/hypervisor.proto

src/runtime/pkg/katautils/config.go

@@ -52,6 +52,7 @@ const (
 	qemuHypervisorTableType        = "qemu"
 	acrnHypervisorTableType        = "acrn"
 	dragonballHypervisorTableType  = "dragonball"
+	remoteHypervisorTableType      = "remote"
 
 	// the maximum amount of PCI bridges that can be cold plugged in a VM
 	maxPCIBridges uint32 = 5
@@ -106,6 +107,7 @@ type hypervisor struct {
 	GuestPreAttestationSecretType  string   `toml:"guest_pre_attestation_secret_type"`
 	SEVCertChainPath               string   `toml:"sev_cert_chain"`
 	BlockDeviceAIO                 string   `toml:"block_device_aio"`
+	RemoteHypervisorSocket         string   `toml:"remote_hypervisor_socket"`
 	HypervisorPathList             []string `toml:"valid_hypervisor_paths"`
 	JailerPathList                 []string `toml:"valid_jailer_paths"`
 	CtlPathList                    []string `toml:"valid_ctlpaths"`
@@ -137,6 +139,7 @@ type hypervisor struct {
 	PCIeRootPort                   uint32   `toml:"pcie_root_port"`
 	GuestPreAttestationGRPCTimeout uint32   `toml:"guest_pre_attestation_grpc_timeout"`
 	SEVGuestPolicy                 uint32   `toml:"sev_guest_policy"`
+	RemoteHypervisorTimeout        uint32   `toml:"remote_hypervisor_timeout"`
 	NumVCPUs                       int32    `toml:"default_vcpus"`
 	BlockDeviceCacheSet            bool     `toml:"block_device_cache_set"`
 	BlockDeviceCacheDirect         bool     `toml:"block_device_cache_direct"`
@@ -1052,6 +1055,17 @@ func newDragonballHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
 	}, nil
 }
 
+func newRemoteHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
+
+	return vc.HypervisorConfig{
+		RemoteHypervisorSocket:  h.RemoteHypervisorSocket,
+		RemoteHypervisorTimeout: h.RemoteHypervisorTimeout,
+
+		// No valid value so avoid to append block device to list in kata_agent.appendDevices
+		BlockDeviceDriver: "dummy",
+	}, nil
+}
+
 func newFactoryConfig(f factory) (oci.FactoryConfig, error) {
 	if f.TemplatePath == "" {
 		f.TemplatePath = defaultTemplatePath
@@ -1088,6 +1102,9 @@ func updateRuntimeConfigHypervisor(configPath string, tomlConf tomlConfig, confi
 		case dragonballHypervisorTableType:
 			config.HypervisorType = vc.DragonballHypervisor
 			hConfig, err = newDragonballHypervisorConfig(hypervisor)
+		case remoteHypervisorTableType:
+			config.HypervisorType = vc.RemoteHypervisor
+			hConfig, err = newRemoteHypervisorConfig(hypervisor)
 		}
 
 		if err != nil {
@@ -1638,6 +1655,11 @@ func checkFactoryConfig(config oci.RuntimeConfig) error {
 // checkHypervisorConfig performs basic "sanity checks" on the hypervisor
 // config.
 func checkHypervisorConfig(config vc.HypervisorConfig) error {
+
+	if config.RemoteHypervisorSocket != "" {
+		return nil
+	}
+
 	type image struct {
 		path   string
 		initrd bool

src/runtime/protocols/hypervisor/hypervisor.proto

@@ -0,0 +1,46 @@
+// (C) Copyright IBM Corp. 2022.
+// SPDX-License-Identifier: Apache-2.0
+
+syntax = "proto3";
+
+package hypervisor;
+
+service Hypervisor {
+	rpc CreateVM(CreateVMRequest) returns (CreateVMResponse) {}
+	rpc StartVM(StartVMRequest) returns (StartVMResponse) {}
+	rpc StopVM(StopVMRequest) returns (StopVMResponse) {}
+	rpc Version(VersionRequest) returns (VersionResponse) {}
+}
+
+
+message VersionRequest {
+	string version = 1;
+}
+
+message VersionResponse {
+	string version = 1;
+}
+
+message CreateVMRequest {
+	string id = 1;
+	map<string, string> annotations = 2;
+	string networkNamespacePath = 3;
+}
+
+message CreateVMResponse {
+	string agentSocketPath = 1;
+}
+
+message StartVMRequest {
+	string id = 1;
+}
+
+message StartVMResponse {
+}
+
+message StopVMRequest {
+	string id = 1;
+}
+
+message StopVMResponse {
+}

src/runtime/virtcontainers/fs_share_linux.go

@@ -11,6 +11,7 @@ import (
 	"context"
 	"encoding/hex"
 	"fmt"
+	"io/fs"
 	"os"
 	"path/filepath"
 	"sync"
@@ -239,23 +240,43 @@ func (f *FilesystemShare) ShareFile(ctx context.Context, c *Container, m *Mount)
 	if !caps.IsFsSharingSupported() {
 		f.Logger().Debug("filesystem sharing is not supported, files will be copied")
 
-		fileInfo, err := os.Stat(m.Source)
+		var ignored bool
-		if err != nil {
+		srcRoot := filepath.Clean(m.Source)
-			return nil, err
+
+		walk := func(srcPath string, d fs.DirEntry, err error) error {
+
+			if err != nil {
+				return err
+			}
+
+			info, err := d.Info()
+			if err != nil {
+				return err
+			}
+
+			if !(info.Mode().IsRegular() || info.Mode().IsDir() || (info.Mode()&os.ModeSymlink) == os.ModeSymlink) {
+				f.Logger().WithField("ignored-file", srcPath).Debug("Ignoring non-regular file as FS sharing not supported")
+				if srcPath == srcRoot {
+					// Ignore the mount if this is not a regular file (excludes socket, device, ...) as it cannot be handled by
+					// a simple copy. But this should not be treated as an error, only as a limitation.
+					ignored = true
+					return filepath.SkipDir
+				}
+				return nil
+			}
+
+			dstPath := filepath.Join(guestPath, srcPath[len(srcRoot):])
+
+			return f.sandbox.agent.copyFile(ctx, srcPath, dstPath)
 		}
 
-		// Ignore the mount if this is not a regular file (excludes
+		if err := filepath.WalkDir(srcRoot, walk); err != nil {
-		// directory, socket, device, ...) as it cannot be handled by
+			c.Logger().WithField("failed-file", m.Source).Debugf("failed to copy file to sandbox: %v", err)
-		// a simple copy. But this should not be treated as an error,
+			return nil, err
-		// only as a limitation.
+		}
-		if !fileInfo.Mode().IsRegular() {
+		if ignored {
-			f.Logger().WithField("ignored-file", m.Source).Debug("Ignoring non-regular file as FS sharing not supported")
 			return nil, nil
 		}
-
-		if err := f.sandbox.agent.copyFile(ctx, m.Source, guestPath); err != nil {
-			return nil, err
-		}
 	} else {
 		// These mounts are created in the shared dir
 		mountDest := filepath.Join(getMountPath(f.sandbox.ID()), filename)

src/runtime/virtcontainers/hypervisor.go

@@ -49,6 +49,9 @@ const (
 	// DragonballHypervisor is the Dragonball hypervisor.
 	DragonballHypervisor HypervisorType = "dragonball"
 
+	// RemoteHypervisor is the Remote hypervisor.
+	RemoteHypervisor HypervisorType = "remote"
+
 	// MockHypervisor is a mock hypervisor for testing purposes
 	MockHypervisor HypervisorType = "mock"
 
@@ -175,6 +178,9 @@ func (hType *HypervisorType) Set(value string) error {
 	case "dragonball":
 		*hType = DragonballHypervisor
 		return nil
+	case "remote":
+		*hType = RemoteHypervisor
+		return nil
 	case "mock":
 		*hType = MockHypervisor
 		return nil
@@ -194,6 +200,8 @@ func (hType *HypervisorType) String() string {
 		return string(AcrnHypervisor)
 	case ClhHypervisor:
 		return string(ClhHypervisor)
+	case RemoteHypervisor:
+		return string(RemoteHypervisor)
 	case MockHypervisor:
 		return string(MockHypervisor)
 	default:
@@ -279,6 +287,9 @@ type HypervisorConfig struct {
 	GuestPreAttestationSecretType  string
 	SEVCertChainPath               string
 	BlockDeviceAIO                 string
+	RemoteHypervisorSocket         string
+	SandboxName                    string
+	SandboxNamespace               string
 	JailerPathList                 []string
 	EntropySourceList              []string
 	VirtioFSDaemonList             []string
@@ -316,6 +327,7 @@ type HypervisorConfig struct {
 	SEVGuestPolicy                 uint32
 	PCIeRootPort                   uint32
 	NumVCPUs                       uint32
+	RemoteHypervisorTimeout        uint32
 	IOMMUPlatform                  bool
 	EnableIOThreads                bool
 	Debug                          bool

src/runtime/virtcontainers/hypervisor_config_darwin.go

@@ -11,6 +11,10 @@ import (
 
 func validateHypervisorConfig(conf *HypervisorConfig) error {
 
+	if conf.RemoteHypervisorSocket != "" {
+		return nil
+	}
+
 	if conf.KernelPath == "" {
 		return fmt.Errorf("Missing kernel path")
 	}

src/runtime/virtcontainers/hypervisor_config_linux.go

@@ -13,6 +13,10 @@ import (
 
 func validateHypervisorConfig(conf *HypervisorConfig) error {
 
+	if conf.RemoteHypervisorSocket != "" {
+		return nil
+	}
+
 	if conf.KernelPath == "" {
 		return fmt.Errorf("Missing kernel path")
 	}

src/runtime/virtcontainers/hypervisor_linux.go

@@ -39,6 +39,8 @@ func NewHypervisor(hType HypervisorType) (Hypervisor, error) {
 		return &cloudHypervisor{}, nil
 	case DragonballHypervisor:
 		return &mockHypervisor{}, nil
+	case RemoteHypervisor:
+		return &remoteHypervisor{}, nil
 	case MockHypervisor:
 		return &mockHypervisor{}, nil
 	default:

src/runtime/virtcontainers/kata_agent.go

@@ -10,6 +10,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"io/ioutil"
 	"os"
 	"path/filepath"
 	"strconv"
@@ -73,9 +74,13 @@ const (
 	kernelParamDebugConsoleVPortValue = "1026"
 )
 
+type customRequestTimeoutKeyType struct{}
+
 var (
 	checkRequestTimeout          = 30 * time.Second
 	defaultRequestTimeout        = 60 * time.Second
+	remoteRequestTimeout         = 300 * time.Second
+	customRequestTimeoutKey      = customRequestTimeoutKeyType(struct{}{})
 	errorMissingOCISpec          = errors.New("Missing OCI specification")
 	defaultKataHostSharedDir     = "/run/kata-containers/shared/sandboxes/"
 	defaultKataGuestSharedDir    = "/run/kata-containers/shared/containers/"
@@ -362,6 +367,8 @@ func (k *kataAgent) agentURL() (string, error) {
 		return s.String(), nil
 	case types.HybridVSock:
 		return s.String(), nil
+	case types.RemoteSock:
+		return s.String(), nil
 	case types.MockHybridVSock:
 		return s.String(), nil
 	default:
@@ -412,6 +419,7 @@ func (k *kataAgent) configure(ctx context.Context, h Hypervisor, id, sharePath s
 		if err != nil {
 			return err
 		}
+	case types.RemoteSock:
 	case types.MockHybridVSock:
 	default:
 		return types.ErrInvalidConfigType
@@ -716,30 +724,38 @@ func (k *kataAgent) startSandbox(ctx context.Context, sandbox *Sandbox) error {
 		return err
 	}
 
-	// Check grpc server is serving
+	var kmodules []*grpc.KernelModule
-	if err = k.check(ctx); err != nil {
-		return err
-	}
 
-	// Setup network interfaces and routes
+	if sandbox.config.HypervisorType == RemoteHypervisor {
-	interfaces, routes, neighs, err := generateVCNetworkStructures(ctx, sandbox.network)
+		ctx = context.WithValue(ctx, customRequestTimeoutKey, remoteRequestTimeout)
-	if err != nil {
+	} else {
-		return err
+		// TODO: Enable the following features for remote hypervisor if necessary
-	}
+
-	if err = k.updateInterfaces(ctx, interfaces); err != nil {
+		// Check grpc server is serving
-		return err
+		if err = k.check(ctx); err != nil {
-	}
+			return err
-	if _, err = k.updateRoutes(ctx, routes); err != nil {
+		}
-		return err
+
-	}
+		// Setup network interfaces and routes
-	if err = k.addARPNeighbors(ctx, neighs); err != nil {
+		interfaces, routes, neighs, err := generateVCNetworkStructures(ctx, sandbox.network)
-		return err
+		if err != nil {
+			return err
+		}
+		if err = k.updateInterfaces(ctx, interfaces); err != nil {
+			return err
+		}
+		if _, err = k.updateRoutes(ctx, routes); err != nil {
+			return err
+		}
+		if err = k.addARPNeighbors(ctx, neighs); err != nil {
+			return err
+		}
+
+		kmodules = setupKernelModules(k.kmodules)
 	}
 
 	storages := setupStorages(ctx, sandbox)
 
-	kmodules := setupKernelModules(k.kmodules)
-
 	req := &grpc.CreateSandboxRequest{
 		Hostname:      hostname,
 		Dns:           dns,
@@ -1118,7 +1134,7 @@ func (k *kataAgent) appendDevices(deviceList []*grpc.Device, c *Container) []*gr
 			kataDevice = k.appendVfioDevice(dev, device, c)
 		}
 
-		if kataDevice == nil {
+		if kataDevice == nil || kataDevice.Type == "" {
 			continue
 		}
 
@@ -1993,7 +2009,12 @@ func (k *kataAgent) getReqContext(ctx context.Context, reqName string) (newCtx c
 	case grpcCheckRequest:
 		newCtx, cancel = context.WithTimeout(ctx, checkRequestTimeout)
 	default:
-		newCtx, cancel = context.WithTimeout(ctx, defaultRequestTimeout)
+		var requestTimeout = defaultRequestTimeout
+
+		if timeout, ok := ctx.Value(customRequestTimeoutKey).(time.Duration); ok {
+			requestTimeout = timeout
+		}
+		newCtx, cancel = context.WithTimeout(ctx, requestTimeout)
 	}
 
 	return newCtx, cancel
@@ -2099,40 +2120,57 @@ func (k *kataAgent) setGuestDateTime(ctx context.Context, tv time.Time) error {
 func (k *kataAgent) copyFile(ctx context.Context, src, dst string) error {
 	var st unix.Stat_t
 
-	err := unix.Stat(src, &st)
+	err := unix.Lstat(src, &st)
 	if err != nil {
 		return fmt.Errorf("Could not get file %s information: %v", src, err)
 	}
 
-	b, err := os.ReadFile(src)
+	cpReq := &grpc.CopyFileRequest{
-	if err != nil {
+		Path:     dst,
-		return fmt.Errorf("Could not read file %s: %v", src, err)
+		DirMode:  uint32(DirMode),
+		FileMode: st.Mode,
+		Uid:      int32(st.Uid),
+		Gid:      int32(st.Gid),
 	}
 
-	fileSize := int64(len(b))
+	var b []byte
+
+	switch sflag := st.Mode & unix.S_IFMT; sflag {
+	case unix.S_IFREG:
+		var err error
+		// TODO: Support incrementail file copying instead of loading whole file into memory
+		b, err = ioutil.ReadFile(src)
+		if err != nil {
+			return fmt.Errorf("Could not read file %s: %v", src, err)
+		}
+		cpReq.FileSize = int64(len(b))
+
+	case unix.S_IFDIR:
+
+	case unix.S_IFLNK:
+		symlink, err := os.Readlink(src)
+		if err != nil {
+			return fmt.Errorf("Could not read symlink %s: %v", src, err)
+		}
+		cpReq.Data = []byte(symlink)
+
+	default:
+		return fmt.Errorf("Unsupported file type: %o", sflag)
+	}
 
 	k.Logger().WithFields(logrus.Fields{
 		"source": src,
 		"dest":   dst,
 	}).Debugf("Copying file from host to guest")
 
-	cpReq := &grpc.CopyFileRequest{
-		Path:     dst,
-		DirMode:  uint32(DirMode),
-		FileMode: uint32(st.Mode),
-		FileSize: fileSize,
-		Uid:      int32(st.Uid),
-		Gid:      int32(st.Gid),
-	}
-
 	// Handle the special case where the file is empty
-	if fileSize == 0 {
+	if cpReq.FileSize == 0 {
-		_, err = k.sendReq(ctx, cpReq)
+		_, err := k.sendReq(ctx, cpReq)
 		return err
 	}
 
 	// Copy file by parts if it's needed
-	remainingBytes := fileSize
+	remainingBytes := cpReq.FileSize
 	offset := int64(0)
 	for remainingBytes > 0 {
 		bytesToCopy := int64(len(b))

src/runtime/virtcontainers/pkg/agent/protocols/client/client.go

@@ -34,6 +34,7 @@ import (
 const (
 	VSockSocketScheme     = "vsock"
 	HybridVSockScheme     = "hvsock"
+	RemoteSockScheme      = "remote"
 	MockHybridVSockScheme = "mock"
 )
 
@@ -237,6 +238,11 @@ func parse(sock string) (string, *url.URL, error) {
 		}
 		hybridVSockPort = uint32(port)
 		grpcAddr = HybridVSockScheme + ":" + hvsocket[0]
+	case RemoteSockScheme:
+		if addr.Host != "" {
+			return "", nil, grpcStatus.Errorf(codes.InvalidArgument, "Invalid remote sock scheme: host address must be empty: %s", sock)
+		}
+		grpcAddr = RemoteSockScheme + ":" + addr.Path
 	// just for tests use.
 	case MockHybridVSockScheme:
 		if addr.Path == "" {
@@ -257,6 +263,8 @@ func agentDialer(addr *url.URL) dialer {
 		return VsockDialer
 	case HybridVSockScheme:
 		return HybridVSockDialer
+	case RemoteSockScheme:
+		return RemoteSockDialer
 	case MockHybridVSockScheme:
 		return MockHybridVSockDialer
 	default:
@@ -437,6 +445,31 @@ func HybridVSockDialer(sock string, timeout time.Duration) (net.Conn, error) {
 	return commonDialer(timeout, dialFunc, timeoutErr)
 }
 
+// RemoteSockDialer dials to an agent in a remote hypervisor sandbox
+func RemoteSockDialer(sock string, timeout time.Duration) (net.Conn, error) {
+
+	s := strings.Split(sock, ":")
+	if !(len(s) == 2 && s[0] == RemoteSockScheme) {
+		return nil, fmt.Errorf("failed to parse remote sock: %q", sock)
+	}
+	socketPath := s[1]
+
+	logrus.Printf("Dialing remote sock: %q %q", socketPath, sock)
+
+	dialFunc := func() (net.Conn, error) {
+		conn, err := net.Dial("unix", socketPath)
+		if err != nil {
+			logrus.Errorf("failed to dial remote sock %q: %v", socketPath, err)
+			return nil, err
+		}
+		return conn, nil
+	}
+
+	timeoutErr := grpcStatus.Errorf(codes.DeadlineExceeded, "timed out connecting to remote sock: %s", socketPath)
+
+	return commonDialer(timeout, dialFunc, timeoutErr)
+}
+
 // just for tests use.
 func MockHybridVSockDialer(sock string, timeout time.Duration) (net.Conn, error) {
 	sock = strings.TrimPrefix(sock, "mock:")

src/runtime/virtcontainers/remote.go

@@ -0,0 +1,298 @@
+// (C) Copyright IBM Corp. 2022.
+// SPDX-License-Identifier: Apache-2.0
+
+package virtcontainers
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"os"
+	"time"
+
+	cri "github.com/containerd/containerd/pkg/cri/annotations"
+	"github.com/containerd/ttrpc"
+	persistapi "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors"
+	pb "github.com/kata-containers/kata-containers/src/runtime/protocols/hypervisor"
+	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+)
+
+const defaultMinTimeout = 60
+
+type remoteHypervisor struct {
+	sandboxID       remoteHypervisorSandboxID
+	agentSocketPath string
+	config          HypervisorConfig
+}
+
+type remoteHypervisorSandboxID string
+
+type remoteService struct {
+	conn   net.Conn
+	client pb.HypervisorService
+}
+
+func openRemoteService(socketPath string) (*remoteService, error) {
+
+	conn, err := net.Dial("unix", socketPath)
+	if err != nil {
+		return nil, fmt.Errorf("failed to connect to remote hypervisor socket: %w", err)
+	}
+
+	ttrpcClient := ttrpc.NewClient(conn)
+
+	client := pb.NewHypervisorClient(ttrpcClient)
+
+	s := &remoteService{
+		conn:   conn,
+		client: client,
+	}
+
+	return s, nil
+}
+
+func (s *remoteService) Close() error {
+	return s.conn.Close()
+}
+
+func (rh *remoteHypervisor) CreateVM(ctx context.Context, id string, network Network, hypervisorConfig *HypervisorConfig) error {
+
+	rh.sandboxID = remoteHypervisorSandboxID(id)
+
+	if err := rh.setConfig(hypervisorConfig); err != nil {
+		return err
+	}
+
+	s, err := openRemoteService(hypervisorConfig.RemoteHypervisorSocket)
+	if err != nil {
+		return err
+	}
+	defer s.Close()
+
+	annotations := map[string]string{}
+	annotations[cri.SandboxName] = hypervisorConfig.SandboxName
+	annotations[cri.SandboxNamespace] = hypervisorConfig.SandboxNamespace
+
+	req := &pb.CreateVMRequest{
+		Id:                   id,
+		Annotations:          annotations,
+		NetworkNamespacePath: network.NetworkID(),
+	}
+
+	res, err := s.client.CreateVM(context.Background(), req)
+	if err != nil {
+		return fmt.Errorf("remote hypervisor call failed: %w", err)
+	}
+
+	if res.AgentSocketPath == "" {
+		return errors.New("remote hypervisor does not return tunnel socket path")
+	}
+
+	rh.agentSocketPath = res.AgentSocketPath
+
+	return nil
+}
+
+func (rh *remoteHypervisor) StartVM(ctx context.Context, timeout int) error {
+
+	minTimeout := defaultMinTimeout
+	if rh.config.RemoteHypervisorTimeout > 0 {
+		minTimeout = int(rh.config.RemoteHypervisorTimeout)
+	}
+
+	if timeout < minTimeout {
+		timeout = minTimeout
+	}
+
+	s, err := openRemoteService(rh.config.RemoteHypervisorSocket)
+	if err != nil {
+		return err
+	}
+	defer s.Close()
+
+	req := &pb.StartVMRequest{
+		Id: string(rh.sandboxID),
+	}
+
+	ctx2, cancel := context.WithTimeout(context.Background(), time.Duration(timeout)*time.Second)
+	defer cancel()
+
+	logrus.Printf("calling remote hypervisor StartVM (timeout: %d)", timeout)
+
+	if _, err := s.client.StartVM(ctx2, req); err != nil {
+		return fmt.Errorf("remote hypervisor call failed: %w", err)
+	}
+
+	return nil
+}
+
+func (rh *remoteHypervisor) AttestVM(ctx context.Context) error {
+	return nil
+}
+
+func (rh *remoteHypervisor) StopVM(ctx context.Context, waitOnly bool) error {
+
+	s, err := openRemoteService(rh.config.RemoteHypervisorSocket)
+	if err != nil {
+		return err
+	}
+	defer s.Close()
+
+	req := &pb.StopVMRequest{
+		Id: string(rh.sandboxID),
+	}
+
+	if _, err := s.client.StopVM(context.Background(), req); err != nil {
+		return fmt.Errorf("remote hypervisor call failed: %w", err)
+	}
+
+	return nil
+}
+
+func (rh *remoteHypervisor) GenerateSocket(id string) (interface{}, error) {
+
+	socketPath := rh.agentSocketPath
+	if len(socketPath) == 0 {
+		return nil, errors.New("failed to generate remote sock: TunnelSocketPath is not set")
+	}
+
+	remoteSock := types.RemoteSock{
+		SandboxID:        id,
+		TunnelSocketPath: socketPath,
+	}
+
+	return remoteSock, nil
+}
+
+func notImplemented(name string) error {
+
+	err := errors.Errorf("%s: not implemented", name)
+
+	logrus.Errorf(err.Error())
+
+	if tracer, ok := err.(interface{ StackTrace() errors.StackTrace }); ok {
+		for _, f := range tracer.StackTrace() {
+			logrus.Errorf("%+s:%d\n", f, f)
+		}
+	}
+
+	return err
+}
+
+func (rh *remoteHypervisor) PauseVM(ctx context.Context) error {
+	panic(notImplemented("PauseVM"))
+}
+
+func (rh *remoteHypervisor) SaveVM() error {
+	panic(notImplemented("SaveVM"))
+}
+
+func (rh *remoteHypervisor) ResumeVM(ctx context.Context) error {
+	panic(notImplemented("ResumeVM"))
+}
+
+func (rh *remoteHypervisor) AddDevice(ctx context.Context, devInfo interface{}, devType DeviceType) error {
+	// TODO
+	logrus.Printf("addDevice: deviceType=%v devInfo=%#v", devType, devInfo)
+	return nil
+}
+
+func (rh *remoteHypervisor) HotplugAddDevice(ctx context.Context, devInfo interface{}, devType DeviceType) (interface{}, error) {
+	logrus.Printf("HotplugAddDevice: devInfo=%#v", devInfo)
+	return "HotplugAddDevice is not implemented", nil
+}
+
+func (rh *remoteHypervisor) HotplugRemoveDevice(ctx context.Context, devInfo interface{}, devType DeviceType) (interface{}, error) {
+	logrus.Printf("HotplugRemoveDevice: devInfo=%#v", devInfo)
+	return "HotplugRemoveDevice is not implemented", nil
+}
+
+func (rh *remoteHypervisor) ResizeMemory(ctx context.Context, memMB uint32, memoryBlockSizeMB uint32, probe bool) (uint32, MemoryDevice, error) {
+	// TODO
+	return memMB, MemoryDevice{}, nil
+}
+
+func (rh *remoteHypervisor) GetTotalMemoryMB(ctx context.Context) uint32 {
+	return rh.config.MemorySize
+}
+
+func (rh *remoteHypervisor) ResizeVCPUs(ctx context.Context, vcpus uint32) (uint32, uint32, error) {
+	// TODO
+	return vcpus, vcpus, nil
+}
+
+func (rh *remoteHypervisor) GetVMConsole(ctx context.Context, sandboxID string) (string, string, error) {
+	panic(notImplemented("GetVMConsole"))
+}
+
+func (rh *remoteHypervisor) Disconnect(ctx context.Context) {
+	// TODO
+	panic(notImplemented("Disconnect"))
+}
+
+func (rh *remoteHypervisor) Capabilities(ctx context.Context) types.Capabilities {
+	var caps types.Capabilities
+	caps.SetBlockDeviceHotplugSupport()
+	return caps
+}
+
+func (rh *remoteHypervisor) HypervisorConfig() HypervisorConfig {
+	return rh.config
+}
+
+func (rh *remoteHypervisor) GetThreadIDs(ctx context.Context) (VcpuThreadIDs, error) {
+	// TODO
+	return VcpuThreadIDs{vcpus: make(map[int]int)}, nil
+}
+
+func (rh *remoteHypervisor) Cleanup(ctx context.Context) error {
+	// TODO
+	return nil
+}
+
+func (rh *remoteHypervisor) setConfig(config *HypervisorConfig) error {
+	// Create a Validator specific for remote hypervisor
+	rh.config = *config
+
+	return nil
+}
+
+func (rh *remoteHypervisor) GetPids() []int {
+	// TODO: meanwhile let's use shim pid as it used by crio to fetch start time
+	return []int{os.Getpid()}
+}
+
+func (rh *remoteHypervisor) GetVirtioFsPid() *int {
+	panic(notImplemented("GetVirtioFsPid"))
+}
+
+func (rh *remoteHypervisor) fromGrpc(ctx context.Context, hypervisorConfig *HypervisorConfig, j []byte) error {
+	panic(notImplemented("fromGrpc"))
+}
+
+func (rh *remoteHypervisor) toGrpc(ctx context.Context) ([]byte, error) {
+	panic(notImplemented("toGrpc"))
+}
+
+func (rh *remoteHypervisor) Check() error {
+	//TODO
+	return nil
+}
+
+func (rh *remoteHypervisor) Save() persistapi.HypervisorState {
+	// TODO
+	// called from Sandbox.dumpHypervisor
+	return persistapi.HypervisorState{}
+}
+
+func (rh *remoteHypervisor) Load(persistapi.HypervisorState) {
+	// TODO
+	// called from Sandbox.loadHypervisor
+}
+
+func (rh *remoteHypervisor) IsRateLimiterBuiltin() bool {
+	// TODO
+	return true
+}

src/runtime/virtcontainers/sandbox.go

@@ -27,6 +27,8 @@ import (
 	"github.com/sirupsen/logrus"
 	"github.com/vishvananda/netlink"
 
+	cri "github.com/containerd/containerd/pkg/cri/annotations"
+	crio "github.com/containers/podman/v4/pkg/annotations"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/api"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/config"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/drivers"
@@ -592,6 +594,21 @@ func newSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Factor
 		return nil, err
 	}
 
+	if len(sandboxConfig.Containers) > 0 {
+		// These values are required by remove hypervisor
+		for _, a := range []string{cri.SandboxName, crio.SandboxName} {
+			if value, ok := sandboxConfig.Containers[0].Annotations[a]; ok {
+				sandboxConfig.HypervisorConfig.SandboxName = value
+			}
+		}
+
+		for _, a := range []string{cri.SandboxNamespace, crio.Namespace} {
+			if value, ok := sandboxConfig.Containers[0].Annotations[a]; ok {
+				sandboxConfig.HypervisorConfig.SandboxNamespace = value
+			}
+		}
+	}
+
 	// store doesn't require hypervisor to be stored immediately
 	if err = s.hypervisor.CreateVM(ctx, s.id, s.network, &sandboxConfig.HypervisorConfig); err != nil {
 		return nil, err

src/runtime/virtcontainers/types/sandbox.go

@@ -7,6 +7,7 @@ package types
 
 import (
 	"fmt"
+	"net"
 	"os"
 	"strings"
 
@@ -37,6 +38,7 @@ const (
 	HybridVSockScheme     = "hvsock"
 	MockHybridVSockScheme = "mock"
 	VSockScheme           = "vsock"
+	RemoteSockScheme      = "remote"
 )
 
 // SandboxState is a sandbox state structure
@@ -210,6 +212,16 @@ func (s *HybridVSock) String() string {
 	return fmt.Sprintf("%s://%s:%d", HybridVSockScheme, s.UdsPath, s.Port)
 }
 
+type RemoteSock struct {
+	Conn             net.Conn
+	SandboxID        string
+	TunnelSocketPath string
+}
+
+func (s *RemoteSock) String() string {
+	return fmt.Sprintf("%s://%s", RemoteSockScheme, s.TunnelSocketPath)
+}
+
 // MockHybridVSock defines a mock hybrid vsocket for tests only.
 type MockHybridVSock struct {
 	UdsPath string