runtime: Support the remote hypervisor type

This patch adds the support of the remote hypervisor type.
Shim opens a Unix domain socket specified in the config file,
and sends TTPRC requests to a external process to control
sandbox VMs.

Fixes #4482

Co-authored-by: Pradipta Banerjee <pradipta.banerjee@gmail.com>
Signed-off-by: Yohei Ueda <yohei@jp.ibm.com>

src/runtime/cmd/kata-runtime/kata-check_amd64.go

@@ -304,6 +304,8 @@ func archHostCanCreateVMContainer(hypervisorType vc.HypervisorType) error {
 		return kvmIsUsable()
 	case "acrn":
 		return acrnIsUsable()
+	case "remote":
+		return nil
 	case "mock":
 		return nil
 	default:

src/runtime/cmd/kata-runtime/kata-check_arm64.go

@@ -86,6 +86,9 @@ func checkKVMExtensions() error {
 }
 
 func archHostCanCreateVMContainer(hypervisorType vc.HypervisorType) error {
+	if hypervisorType == "remote" {
+		return nil
+	}
 	if err := kvmIsUsable(); err != nil {
 		return err
 	}

src/runtime/cmd/kata-runtime/kata-check_ppc64le.go

@@ -61,6 +61,9 @@ func setCPUtype(hypervisorType vc.HypervisorType) error {
 }
 
 func archHostCanCreateVMContainer(hypervisorType vc.HypervisorType) error {
+	if hypervisorType == "remote" {
+		return nil
+	}
 	return kvmIsUsable()
 }
 

src/runtime/cmd/kata-runtime/kata-check_s390x.go

@@ -55,6 +55,9 @@ func kvmIsUsable() error {
 }
 
 func archHostCanCreateVMContainer(hypervisorType vc.HypervisorType) error {
+	if hypervisorType == "remote" {
+		return nil
+	}
 	return kvmIsUsable()
 }
 

src/runtime/hack/update-generated-hypervisor-proto.sh

@@ -1,4 +1,6 @@
 #!/bin/bash
+# (C) Copyright IBM Corp. 2022.
+# SPDX-License-Identifier: Apache-2.0
 
 set -o errexit -o pipefail -o nounset
 
@@ -7,5 +9,5 @@ cd "$(dirname "${BASH_SOURCE[0]}")/.."
 protoc --gogottrpc_out=protocols/hypervisor \
 	--gogottrpc_opt=plugins=ttrpc+fieldpath,paths=source_relative \
 	-Iprotocols/hypervisor \
-	-I../agent/protocols/protos/gogo/protobuf \
+	-I../libs/protocols/protos/gogo/protobuf \
 	protocols/hypervisor/hypervisor.proto

src/runtime/pkg/katautils/config.go

@@ -52,6 +52,7 @@ const (
 	qemuHypervisorTableType        = "qemu"
 	acrnHypervisorTableType        = "acrn"
 	dragonballHypervisorTableType  = "dragonball"
+	remoteHypervisorTableType      = "remote"
 
 	// the maximum amount of PCI bridges that can be cold plugged in a VM
 	maxPCIBridges uint32 = 5
@@ -106,6 +107,7 @@ type hypervisor struct {
 	GuestPreAttestationSecretType  string   `toml:"guest_pre_attestation_secret_type"`
 	SEVCertChainPath               string   `toml:"sev_cert_chain"`
 	BlockDeviceAIO                 string   `toml:"block_device_aio"`
+	RemoteHypervisorSocket         string   `toml:"remote_hypervisor_socket"`
 	HypervisorPathList             []string `toml:"valid_hypervisor_paths"`
 	JailerPathList                 []string `toml:"valid_jailer_paths"`
 	CtlPathList                    []string `toml:"valid_ctlpaths"`
@@ -137,6 +139,7 @@ type hypervisor struct {
 	PCIeRootPort                   uint32   `toml:"pcie_root_port"`
 	GuestPreAttestationGRPCTimeout uint32   `toml:"guest_pre_attestation_grpc_timeout"`
 	SEVGuestPolicy                 uint32   `toml:"sev_guest_policy"`
+	RemoteHypervisorTimeout        uint32   `toml:"remote_hypervisor_timeout"`
 	NumVCPUs                       int32    `toml:"default_vcpus"`
 	BlockDeviceCacheSet            bool     `toml:"block_device_cache_set"`
 	BlockDeviceCacheDirect         bool     `toml:"block_device_cache_direct"`
@@ -1052,6 +1055,14 @@ func newDragonballHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
 	}, nil
 }
 
+func newRemoteHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
+
+	return vc.HypervisorConfig{
+		RemoteHypervisorSocket:  h.RemoteHypervisorSocket,
+		RemoteHypervisorTimeout: h.RemoteHypervisorTimeout,
+	}, nil
+}
+
 func newFactoryConfig(f factory) (oci.FactoryConfig, error) {
 	if f.TemplatePath == "" {
 		f.TemplatePath = defaultTemplatePath
@@ -1088,6 +1099,9 @@ func updateRuntimeConfigHypervisor(configPath string, tomlConf tomlConfig, confi
 		case dragonballHypervisorTableType:
 			config.HypervisorType = vc.DragonballHypervisor
 			hConfig, err = newDragonballHypervisorConfig(hypervisor)
+		case remoteHypervisorTableType:
+			config.HypervisorType = vc.RemoteHypervisor
+			hConfig, err = newRemoteHypervisorConfig(hypervisor)
 		}
 
 		if err != nil {
@@ -1638,6 +1652,11 @@ func checkFactoryConfig(config oci.RuntimeConfig) error {
 // checkHypervisorConfig performs basic "sanity checks" on the hypervisor
 // config.
 func checkHypervisorConfig(config vc.HypervisorConfig) error {
+
+	if config.RemoteHypervisorSocket != "" {
+		return nil
+	}
+
 	type image struct {
 		path   string
 		initrd bool

src/runtime/protocols/hypervisor/hypervisor.proto

@@ -1,3 +1,6 @@
+// (C) Copyright IBM Corp. 2022.
+// SPDX-License-Identifier: Apache-2.0
+
 syntax = "proto3";
 
 package hypervisor;

src/runtime/virtcontainers/hypervisor.go

@@ -49,6 +49,9 @@ const (
 	// DragonballHypervisor is the Dragonball hypervisor.
 	DragonballHypervisor HypervisorType = "dragonball"
 
+	// RemoteHypervisor is the Remote hypervisor.
+	RemoteHypervisor HypervisorType = "remote"
+
 	// MockHypervisor is a mock hypervisor for testing purposes
 	MockHypervisor HypervisorType = "mock"
 
@@ -175,6 +178,9 @@ func (hType *HypervisorType) Set(value string) error {
 	case "dragonball":
 		*hType = DragonballHypervisor
 		return nil
+	case "remote":
+		*hType = RemoteHypervisor
+		return nil
 	case "mock":
 		*hType = MockHypervisor
 		return nil
@@ -194,6 +200,8 @@ func (hType *HypervisorType) String() string {
 		return string(AcrnHypervisor)
 	case ClhHypervisor:
 		return string(ClhHypervisor)
+	case RemoteHypervisor:
+		return string(RemoteHypervisor)
 	case MockHypervisor:
 		return string(MockHypervisor)
 	default:
@@ -279,6 +287,9 @@ type HypervisorConfig struct {
 	GuestPreAttestationSecretType  string
 	SEVCertChainPath               string
 	BlockDeviceAIO                 string
+	RemoteHypervisorSocket         string
+	SandboxName                    string
+	SandboxNamespace               string
 	JailerPathList                 []string
 	EntropySourceList              []string
 	VirtioFSDaemonList             []string
@@ -316,6 +327,7 @@ type HypervisorConfig struct {
 	SEVGuestPolicy                 uint32
 	PCIeRootPort                   uint32
 	NumVCPUs                       uint32
+	RemoteHypervisorTimeout        uint32
 	IOMMUPlatform                  bool
 	EnableIOThreads                bool
 	Debug                          bool

src/runtime/virtcontainers/hypervisor_config_darwin.go

@@ -11,6 +11,10 @@ import (
 
 func validateHypervisorConfig(conf *HypervisorConfig) error {
 
+	if conf.RemoteHypervisorSocket != "" {
+		return nil
+	}
+
 	if conf.KernelPath == "" {
 		return fmt.Errorf("Missing kernel path")
 	}

src/runtime/virtcontainers/hypervisor_config_linux.go

@@ -13,6 +13,10 @@ import (
 
 func validateHypervisorConfig(conf *HypervisorConfig) error {
 
+	if conf.RemoteHypervisorSocket != "" {
+		return nil
+	}
+
 	if conf.KernelPath == "" {
 		return fmt.Errorf("Missing kernel path")
 	}

src/runtime/virtcontainers/hypervisor_linux.go

@@ -39,6 +39,8 @@ func NewHypervisor(hType HypervisorType) (Hypervisor, error) {
 		return &cloudHypervisor{}, nil
 	case DragonballHypervisor:
 		return &mockHypervisor{}, nil
+	case RemoteHypervisor:
+		return &remoteHypervisor{}, nil
 	case MockHypervisor:
 		return &mockHypervisor{}, nil
 	default:

src/runtime/virtcontainers/kata_agent.go

@@ -73,9 +73,13 @@ const (
 	kernelParamDebugConsoleVPortValue = "1026"
 )
 
+type customRequestTimeoutKeyType struct{}
+
 var (
 	checkRequestTimeout          = 30 * time.Second
 	defaultRequestTimeout        = 60 * time.Second
+	remoteRequestTimeout         = 300 * time.Second
+	customRequestTimeoutKey      = customRequestTimeoutKeyType(struct{}{})
 	errorMissingOCISpec          = errors.New("Missing OCI specification")
 	defaultKataHostSharedDir     = "/run/kata-containers/shared/sandboxes/"
 	defaultKataGuestSharedDir    = "/run/kata-containers/shared/containers/"
@@ -362,6 +366,8 @@ func (k *kataAgent) agentURL() (string, error) {
 		return s.String(), nil
 	case types.HybridVSock:
 		return s.String(), nil
+	case types.RemoteSock:
+		return s.String(), nil
 	case types.MockHybridVSock:
 		return s.String(), nil
 	default:
@@ -412,6 +418,7 @@ func (k *kataAgent) configure(ctx context.Context, h Hypervisor, id, sharePath s
 		if err != nil {
 			return err
 		}
+	case types.RemoteSock:
 	case types.MockHybridVSock:
 	default:
 		return types.ErrInvalidConfigType
@@ -716,30 +723,38 @@ func (k *kataAgent) startSandbox(ctx context.Context, sandbox *Sandbox) error {
 		return err
 	}
 
-	// Check grpc server is serving
+	var kmodules []*grpc.KernelModule
-	if err = k.check(ctx); err != nil {
-		return err
-	}
 
-	// Setup network interfaces and routes
+	if sandbox.config.HypervisorType == RemoteHypervisor {
-	interfaces, routes, neighs, err := generateVCNetworkStructures(ctx, sandbox.network)
+		ctx = context.WithValue(ctx, customRequestTimeoutKey, remoteRequestTimeout)
-	if err != nil {
+	} else {
-		return err
+		// TODO: Enable the following features for remote hypervisor if necessary
-	}
+
-	if err = k.updateInterfaces(ctx, interfaces); err != nil {
+		// Check grpc server is serving
-		return err
+		if err = k.check(ctx); err != nil {
-	}
+			return err
-	if _, err = k.updateRoutes(ctx, routes); err != nil {
+		}
-		return err
+
-	}
+		// Setup network interfaces and routes
-	if err = k.addARPNeighbors(ctx, neighs); err != nil {
+		interfaces, routes, neighs, err := generateVCNetworkStructures(ctx, sandbox.network)
-		return err
+		if err != nil {
+			return err
+		}
+		if err = k.updateInterfaces(ctx, interfaces); err != nil {
+			return err
+		}
+		if _, err = k.updateRoutes(ctx, routes); err != nil {
+			return err
+		}
+		if err = k.addARPNeighbors(ctx, neighs); err != nil {
+			return err
+		}
+
+		kmodules = setupKernelModules(k.kmodules)
 	}
 
 	storages := setupStorages(ctx, sandbox)
 
-	kmodules := setupKernelModules(k.kmodules)
-
 	req := &grpc.CreateSandboxRequest{
 		Hostname:      hostname,
 		Dns:           dns,
@@ -1993,7 +2008,12 @@ func (k *kataAgent) getReqContext(ctx context.Context, reqName string) (newCtx c
 	case grpcCheckRequest:
 		newCtx, cancel = context.WithTimeout(ctx, checkRequestTimeout)
 	default:
-		newCtx, cancel = context.WithTimeout(ctx, defaultRequestTimeout)
+		var requestTimeout = defaultRequestTimeout
+
+		if timeout, ok := ctx.Value(customRequestTimeoutKey).(time.Duration); ok {
+			requestTimeout = timeout
+		}
+		newCtx, cancel = context.WithTimeout(ctx, requestTimeout)
 	}
 
 	return newCtx, cancel

src/runtime/virtcontainers/pkg/agent/protocols/client/client.go

@@ -34,6 +34,7 @@ import (
 const (
 	VSockSocketScheme     = "vsock"
 	HybridVSockScheme     = "hvsock"
+	RemoteSockScheme      = "remote"
 	MockHybridVSockScheme = "mock"
 )
 
@@ -237,6 +238,11 @@ func parse(sock string) (string, *url.URL, error) {
 		}
 		hybridVSockPort = uint32(port)
 		grpcAddr = HybridVSockScheme + ":" + hvsocket[0]
+	case RemoteSockScheme:
+		if addr.Host != "" {
+			return "", nil, grpcStatus.Errorf(codes.InvalidArgument, "Invalid remote sock scheme: host address must be empty: %s", sock)
+		}
+		grpcAddr = RemoteSockScheme + ":" + addr.Path
 	// just for tests use.
 	case MockHybridVSockScheme:
 		if addr.Path == "" {
@@ -257,6 +263,8 @@ func agentDialer(addr *url.URL) dialer {
 		return VsockDialer
 	case HybridVSockScheme:
 		return HybridVSockDialer
+	case RemoteSockScheme:
+		return RemoteSockDialer
 	case MockHybridVSockScheme:
 		return MockHybridVSockDialer
 	default:
@@ -437,6 +445,31 @@ func HybridVSockDialer(sock string, timeout time.Duration) (net.Conn, error) {
 	return commonDialer(timeout, dialFunc, timeoutErr)
 }
 
+// RemoteSockDialer dials to an agent in a remote hypervisor sandbox
+func RemoteSockDialer(sock string, timeout time.Duration) (net.Conn, error) {
+
+	s := strings.Split(sock, ":")
+	if !(len(s) == 2 && s[0] == RemoteSockScheme) {
+		return nil, fmt.Errorf("failed to parse remote sock: %q", sock)
+	}
+	socketPath := s[1]
+
+	logrus.Printf("Dialing remote sock: %q %q", socketPath, sock)
+
+	dialFunc := func() (net.Conn, error) {
+		conn, err := net.Dial("unix", socketPath)
+		if err != nil {
+			logrus.Errorf("failed to dial remote sock %q: %v", socketPath, err)
+			return nil, err
+		}
+		return conn, nil
+	}
+
+	timeoutErr := grpcStatus.Errorf(codes.DeadlineExceeded, "timed out connecting to remote sock: %s", socketPath)
+
+	return commonDialer(timeout, dialFunc, timeoutErr)
+}
+
 // just for tests use.
 func MockHybridVSockDialer(sock string, timeout time.Duration) (net.Conn, error) {
 	sock = strings.TrimPrefix(sock, "mock:")

src/runtime/virtcontainers/remote.go

@@ -0,0 +1,296 @@
+// (C) Copyright IBM Corp. 2022.
+// SPDX-License-Identifier: Apache-2.0
+
+package virtcontainers
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"os"
+	"time"
+
+	cri "github.com/containerd/containerd/pkg/cri/annotations"
+	"github.com/containerd/ttrpc"
+	persistapi "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors"
+	pb "github.com/kata-containers/kata-containers/src/runtime/protocols/hypervisor"
+	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+)
+
+const defaultMinTimeout = 60
+
+type remoteHypervisor struct {
+	sandboxID       remoteHypervisorSandboxID
+	agentSocketPath string
+	config          HypervisorConfig
+}
+
+type remoteHypervisorSandboxID string
+
+type remoteService struct {
+	conn   net.Conn
+	client pb.HypervisorService
+}
+
+func openRemoteService(socketPath string) (*remoteService, error) {
+
+	conn, err := net.Dial("unix", socketPath)
+	if err != nil {
+		return nil, fmt.Errorf("failed to connect to remote hypervisor socket: %w", err)
+	}
+
+	ttrpcClient := ttrpc.NewClient(conn)
+
+	client := pb.NewHypervisorClient(ttrpcClient)
+
+	s := &remoteService{
+		conn:   conn,
+		client: client,
+	}
+
+	return s, nil
+}
+
+func (s *remoteService) Close() error {
+	return s.conn.Close()
+}
+
+func (rh *remoteHypervisor) CreateVM(ctx context.Context, id string, network Network, hypervisorConfig *HypervisorConfig) error {
+
+	rh.sandboxID = remoteHypervisorSandboxID(id)
+
+	if err := rh.setConfig(hypervisorConfig); err != nil {
+		return err
+	}
+
+	s, err := openRemoteService(hypervisorConfig.RemoteHypervisorSocket)
+	if err != nil {
+		return err
+	}
+	defer s.Close()
+
+	annotations := map[string]string{}
+	annotations[cri.SandboxName] = hypervisorConfig.SandboxName
+	annotations[cri.SandboxNamespace] = hypervisorConfig.SandboxNamespace
+
+	req := &pb.CreateVMRequest{
+		Id:                   id,
+		Annotations:          annotations,
+		NetworkNamespacePath: network.NetworkID(),
+	}
+
+	res, err := s.client.CreateVM(context.Background(), req)
+	if err != nil {
+		return fmt.Errorf("remote hypervisor call failed: %w", err)
+	}
+
+	if res.AgentSocketPath == "" {
+		return errors.New("remote hypervisor does not return tunnel socket path")
+	}
+
+	rh.agentSocketPath = res.AgentSocketPath
+
+	return nil
+}
+
+func (rh *remoteHypervisor) StartVM(ctx context.Context, timeout int) error {
+
+	minTimeout := defaultMinTimeout
+	if rh.config.RemoteHypervisorTimeout > 0 {
+		minTimeout = int(rh.config.RemoteHypervisorTimeout)
+	}
+
+	if timeout < minTimeout {
+		timeout = minTimeout
+	}
+
+	s, err := openRemoteService(rh.config.RemoteHypervisorSocket)
+	if err != nil {
+		return err
+	}
+	defer s.Close()
+
+	req := &pb.StartVMRequest{
+		Id: string(rh.sandboxID),
+	}
+
+	ctx2, cancel := context.WithTimeout(context.Background(), time.Duration(timeout)*time.Second)
+	defer cancel()
+
+	logrus.Printf("calling remote hypervisor StartVM (timeout: %d)", timeout)
+
+	if _, err := s.client.StartVM(ctx2, req); err != nil {
+		return fmt.Errorf("remote hypervisor call failed: %w", err)
+	}
+
+	return nil
+}
+
+func (rh *remoteHypervisor) AttestVM(ctx context.Context) error {
+	return nil
+}
+
+func (rh *remoteHypervisor) StopVM(ctx context.Context, waitOnly bool) error {
+
+	s, err := openRemoteService(rh.config.RemoteHypervisorSocket)
+	if err != nil {
+		return err
+	}
+	defer s.Close()
+
+	req := &pb.StopVMRequest{
+		Id: string(rh.sandboxID),
+	}
+
+	if _, err := s.client.StopVM(context.Background(), req); err != nil {
+		return fmt.Errorf("remote hypervisor call failed: %w", err)
+	}
+
+	return nil
+}
+
+func (rh *remoteHypervisor) GenerateSocket(id string) (interface{}, error) {
+
+	socketPath := rh.agentSocketPath
+	if len(socketPath) == 0 {
+		return nil, errors.New("failed to generate remote sock: TunnelSocketPath is not set")
+	}
+
+	remoteSock := types.RemoteSock{
+		SandboxID:        id,
+		TunnelSocketPath: socketPath,
+	}
+
+	return remoteSock, nil
+}
+
+func notImplemented(name string) error {
+
+	err := errors.Errorf("%s: not implemented", name)
+
+	logrus.Errorf(err.Error())
+
+	if tracer, ok := err.(interface{ StackTrace() errors.StackTrace }); ok {
+		for _, f := range tracer.StackTrace() {
+			logrus.Errorf("%+s:%d\n", f, f)
+		}
+	}
+
+	return err
+}
+
+func (rh *remoteHypervisor) PauseVM(ctx context.Context) error {
+	panic(notImplemented("PauseVM"))
+}
+
+func (rh *remoteHypervisor) SaveVM() error {
+	panic(notImplemented("SaveVM"))
+}
+
+func (rh *remoteHypervisor) ResumeVM(ctx context.Context) error {
+	panic(notImplemented("ResumeVM"))
+}
+
+func (rh *remoteHypervisor) AddDevice(ctx context.Context, devInfo interface{}, devType DeviceType) error {
+	// TODO
+	logrus.Printf("addDevice: deviceType=%v devInfo=%#v", devType, devInfo)
+	return nil
+}
+
+func (rh *remoteHypervisor) HotplugAddDevice(ctx context.Context, devInfo interface{}, devType DeviceType) (interface{}, error) {
+	panic(notImplemented("HotplugAddDevice"))
+}
+
+func (rh *remoteHypervisor) HotplugRemoveDevice(ctx context.Context, devInfo interface{}, devType DeviceType) (interface{}, error) {
+	panic(notImplemented("HotplugRemoveDevice"))
+}
+
+func (rh *remoteHypervisor) ResizeMemory(ctx context.Context, memMB uint32, memoryBlockSizeMB uint32, probe bool) (uint32, MemoryDevice, error) {
+	// TODO
+	return memMB, MemoryDevice{}, nil
+}
+
+func (rh *remoteHypervisor) GetTotalMemoryMB(ctx context.Context) uint32 {
+	return rh.config.MemorySize
+}
+
+func (rh *remoteHypervisor) ResizeVCPUs(ctx context.Context, vcpus uint32) (uint32, uint32, error) {
+	// TODO
+	return vcpus, vcpus, nil
+}
+
+func (rh *remoteHypervisor) GetVMConsole(ctx context.Context, sandboxID string) (string, string, error) {
+	panic(notImplemented("GetVMConsole"))
+}
+
+func (rh *remoteHypervisor) Disconnect(ctx context.Context) {
+	// TODO
+	panic(notImplemented("Disconnect"))
+}
+
+func (rh *remoteHypervisor) Capabilities(ctx context.Context) types.Capabilities {
+	var caps types.Capabilities
+	caps.SetBlockDeviceHotplugSupport()
+	return caps
+}
+
+func (rh *remoteHypervisor) HypervisorConfig() HypervisorConfig {
+	return rh.config
+}
+
+func (rh *remoteHypervisor) GetThreadIDs(ctx context.Context) (VcpuThreadIDs, error) {
+	// TODO
+	return VcpuThreadIDs{vcpus: make(map[int]int)}, nil
+}
+
+func (rh *remoteHypervisor) Cleanup(ctx context.Context) error {
+	// TODO
+	return nil
+}
+
+func (rh *remoteHypervisor) setConfig(config *HypervisorConfig) error {
+	// Create a Validator specific for remote hypervisor
+	rh.config = *config
+
+	return nil
+}
+
+func (rh *remoteHypervisor) GetPids() []int {
+	// TODO: meanwhile let's use shim pid as it used by crio to fetch start time
+	return []int{os.Getpid()}
+}
+
+func (rh *remoteHypervisor) GetVirtioFsPid() *int {
+	panic(notImplemented("GetVirtioFsPid"))
+}
+
+func (rh *remoteHypervisor) fromGrpc(ctx context.Context, hypervisorConfig *HypervisorConfig, j []byte) error {
+	panic(notImplemented("fromGrpc"))
+}
+
+func (rh *remoteHypervisor) toGrpc(ctx context.Context) ([]byte, error) {
+	panic(notImplemented("toGrpc"))
+}
+
+func (rh *remoteHypervisor) Check() error {
+	//TODO
+	return nil
+}
+
+func (rh *remoteHypervisor) Save() persistapi.HypervisorState {
+	// TODO
+	// called from Sandbox.dumpHypervisor
+	return persistapi.HypervisorState{}
+}
+
+func (rh *remoteHypervisor) Load(persistapi.HypervisorState) {
+	// TODO
+	// called from Sandbox.loadHypervisor
+}
+
+func (rh *remoteHypervisor) IsRateLimiterBuiltin() bool {
+	// TODO
+	return true
+}

src/runtime/virtcontainers/sandbox.go

@@ -27,6 +27,8 @@ import (
 	"github.com/sirupsen/logrus"
 	"github.com/vishvananda/netlink"
 
+	cri "github.com/containerd/containerd/pkg/cri/annotations"
+	crio "github.com/containers/podman/v4/pkg/annotations"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/api"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/config"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/drivers"
@@ -592,6 +594,21 @@ func newSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Factor
 		return nil, err
 	}
 
+	if len(sandboxConfig.Containers) > 0 {
+		// These values are required by remove hypervisor
+		for _, a := range []string{cri.SandboxName, crio.SandboxName} {
+			if value, ok := sandboxConfig.Containers[0].Annotations[a]; ok {
+				sandboxConfig.HypervisorConfig.SandboxName = value
+			}
+		}
+
+		for _, a := range []string{cri.SandboxNamespace, crio.Namespace} {
+			if value, ok := sandboxConfig.Containers[0].Annotations[a]; ok {
+				sandboxConfig.HypervisorConfig.SandboxNamespace = value
+			}
+		}
+	}
+
 	// store doesn't require hypervisor to be stored immediately
 	if err = s.hypervisor.CreateVM(ctx, s.id, s.network, &sandboxConfig.HypervisorConfig); err != nil {
 		return nil, err

src/runtime/virtcontainers/types/sandbox.go

@@ -7,6 +7,7 @@ package types
 
 import (
 	"fmt"
+	"net"
 	"os"
 	"strings"
 
@@ -37,6 +38,7 @@ const (
 	HybridVSockScheme     = "hvsock"
 	MockHybridVSockScheme = "mock"
 	VSockScheme           = "vsock"
+	RemoteSockScheme      = "remote"
 )
 
 // SandboxState is a sandbox state structure
@@ -210,6 +212,16 @@ func (s *HybridVSock) String() string {
 	return fmt.Sprintf("%s://%s:%d", HybridVSockScheme, s.UdsPath, s.Port)
 }
 
+type RemoteSock struct {
+	Conn             net.Conn
+	SandboxID        string
+	TunnelSocketPath string
+}
+
+func (s *RemoteSock) String() string {
+	return fmt.Sprintf("%s://%s", RemoteSockScheme, s.TunnelSocketPath)
+}
+
 // MockHybridVSock defines a mock hybrid vsocket for tests only.
 type MockHybridVSock struct {
 	UdsPath string