runtime: Show socket path in kata-env output

Display a pseudo path to the sandbox socket in the output of
`kata-runtime env` for those hypervisors that use Hybrid VSOCK.

The path is not a real path since the command does not create a sandbox.
The output includes a `{ID}` tag which would be replaced with the real
sandbox ID (name) when the sandbox was created.

This feature is only useful for agent tracing with the trace forwarder
where the configured hypervisor uses Hybrid VSOCK.

Note that the features required a new `setConfig()` method to be added
to the `hypervisor` interface. This isn't normally needed as the
specified hypervisor configuration passed to `setConfig()` is also
passed to `createSandbox()`. However the new call is required by
`kata-runtime env` to display the correct socket path for Firecracker.
The new method isn't wholly redundant for the main code path though as
it's now used by each hypervisor's `createSandbox()` call.

Signed-off-by: James O. D. Hunt <james.o.hunt@intel.com>
This commit is contained in:
James O. D. Hunt 2021-10-11 14:33:30 +01:00
parent 5b3a349db5
commit e61f5e2931
15 changed files with 324 additions and 50 deletions

View File

@ -29,7 +29,7 @@ import (
//
// XXX: Increment for every change to the output format
// (meaning any change to the EnvInfo type).
const formatVersion = "1.0.25"
const formatVersion = "1.0.26"
// MetaInfo stores information on the format of the output itself
type MetaInfo struct {
@ -108,6 +108,7 @@ type HypervisorInfo struct {
EntropySource string
SharedFS string
VirtioFSDaemon string
SocketPath string
Msize9p uint32
MemorySlots uint32
PCIeRootPort uint32
@ -305,7 +306,7 @@ func getAgentInfo(config oci.RuntimeConfig) (AgentInfo, error) {
return agent, nil
}
func getHypervisorInfo(config oci.RuntimeConfig) HypervisorInfo {
func getHypervisorInfo(config oci.RuntimeConfig) (HypervisorInfo, error) {
hypervisorPath := config.HypervisorConfig.HypervisorPath
version, err := getCommandVersion(hypervisorPath)
@ -313,6 +314,19 @@ func getHypervisorInfo(config oci.RuntimeConfig) HypervisorInfo {
version = unknown
}
hypervisorType := config.HypervisorType
socketPath := unknown
// It is only reliable to make this call as root since a
// non-privileged user may not have access to /dev/vhost-vsock.
if os.Geteuid() == 0 {
socketPath, err = vc.GetHypervisorSocketTemplate(hypervisorType, &config.HypervisorConfig)
if err != nil {
return HypervisorInfo{}, err
}
}
return HypervisorInfo{
Debug: config.HypervisorConfig.Debug,
MachineType: config.HypervisorConfig.HypervisorMachineType,
@ -327,7 +341,8 @@ func getHypervisorInfo(config oci.RuntimeConfig) HypervisorInfo {
HotplugVFIOOnRootBus: config.HypervisorConfig.HotplugVFIOOnRootBus,
PCIeRootPort: config.HypervisorConfig.PCIeRootPort,
}
SocketPath: socketPath,
}, nil
}
func getEnvInfo(configFile string, config oci.RuntimeConfig) (env EnvInfo, err error) {
@ -352,7 +367,10 @@ func getEnvInfo(configFile string, config oci.RuntimeConfig) (env EnvInfo, err e
return EnvInfo{}, err
}
hypervisor := getHypervisorInfo(config)
hypervisor, err := getHypervisorInfo(config)
if err != nil {
return EnvInfo{}, err
}
image := ImageInfo{
Path: config.HypervisorConfig.ImagePath,

View File

@ -277,7 +277,7 @@ VERSION_ID="%s"
}
func getExpectedHypervisor(config oci.RuntimeConfig) HypervisorInfo {
return HypervisorInfo{
info := HypervisorInfo{
Version: testHypervisorVersion,
Path: config.HypervisorConfig.HypervisorPath,
MachineType: config.HypervisorConfig.HypervisorMachineType,
@ -292,6 +292,16 @@ func getExpectedHypervisor(config oci.RuntimeConfig) HypervisorInfo {
HotplugVFIOOnRootBus: config.HypervisorConfig.HotplugVFIOOnRootBus,
PCIeRootPort: config.HypervisorConfig.PCIeRootPort,
}
if os.Geteuid() == 0 {
// This assumes the test hypervisor is a non-hybrid-vsock
// one (such as QEMU).
info.SocketPath = ""
} else {
info.SocketPath = unknown
}
return info
}
func getExpectedImage(config oci.RuntimeConfig) ImageInfo {
@ -1007,12 +1017,58 @@ func TestGetHypervisorInfo(t *testing.T) {
_, config, err := makeRuntimeConfig(tmpdir)
assert.NoError(err)
info := getHypervisorInfo(config)
info, err := getHypervisorInfo(config)
assert.NoError(err)
assert.Equal(info.Version, testHypervisorVersion)
err = os.Remove(config.HypervisorConfig.HypervisorPath)
assert.NoError(err)
info = getHypervisorInfo(config)
info, err = getHypervisorInfo(config)
assert.NoError(err)
assert.Equal(info.Version, unknown)
}
func TestGetHypervisorInfoSocket(t *testing.T) {
assert := assert.New(t)
tmpdir, err := ioutil.TempDir("", "")
assert.NoError(err)
defer os.RemoveAll(tmpdir)
_, config, err := makeRuntimeConfig(tmpdir)
assert.NoError(err)
type TestHypervisorDetails struct {
hType vc.HypervisorType
hybridVsock bool
}
hypervisors := []TestHypervisorDetails{
{vc.AcrnHypervisor, false},
{vc.ClhHypervisor, true},
{vc.FirecrackerHypervisor, true},
{vc.MockHypervisor, false},
{vc.QemuHypervisor, false},
}
for i, details := range hypervisors {
msg := fmt.Sprintf("hypervisor[%d]: %+v", i, details)
config.HypervisorType = details.hType
info, err := getHypervisorInfo(config)
assert.NoError(err, msg)
if os.Geteuid() == 0 {
if !details.hybridVsock {
assert.Equal(info.SocketPath, "", msg)
} else {
assert.NotEmpty(info.SocketPath, msg)
assert.True(strings.HasPrefix(info.SocketPath, "/"), msg)
}
} else {
assert.Equal(info.SocketPath, unknown, msg)
}
}
}

View File

@ -284,13 +284,11 @@ func (a *Acrn) setup(ctx context.Context, id string, hypervisorConfig *Hyperviso
span, _ := katatrace.Trace(ctx, a.Logger(), "setup", acrnTracingTags, map[string]string{"sandbox_id": a.id})
defer span.End()
err := hypervisorConfig.valid()
if err != nil {
if err := a.setConfig(hypervisorConfig); err != nil {
return err
}
a.id = id
a.config = *hypervisorConfig
a.arch = newAcrnArch(a.config)
var create bool
@ -302,6 +300,9 @@ func (a *Acrn) setup(ctx context.Context, id string, hypervisorConfig *Hyperviso
if create {
a.Logger().Debug("Setting UUID")
var err error
if uuid, err = a.GetNextAvailableUUID(); err != nil {
return err
}
@ -342,6 +343,16 @@ func (a *Acrn) createDummyVirtioBlkDev(ctx context.Context, devices []Device) ([
return devices, nil
}
func (a *Acrn) setConfig(config *HypervisorConfig) error {
if err := config.valid(); err != nil {
return err
}
a.config = *config
return nil
}
// createSandbox is the Hypervisor sandbox creation.
func (a *Acrn) createSandbox(ctx context.Context, id string, networkNS NetworkNamespace, hypervisorConfig *HypervisorConfig) error {
// Save the tracing context

View File

@ -259,3 +259,18 @@ func TestAcrnMemoryTopology(t *testing.T) {
assert.NoError(err)
assert.Exactly(memory, expectedOut)
}
func TestAcrnSetConfig(t *testing.T) {
assert := assert.New(t)
config := newAcrnConfig()
a := &Acrn{}
assert.Equal(a.config, HypervisorConfig{})
err := a.setConfig(&config)
assert.NoError(err)
assert.Equal(a.config, config)
}

View File

@ -188,6 +188,17 @@ var clhDebugKernelParams = []Param{
//
//###########################################################
func (clh *cloudHypervisor) setConfig(config *HypervisorConfig) error {
err := config.valid()
if err != nil {
return err
}
clh.config = *config
return nil
}
// For cloudHypervisor this call only sets the internal structure up.
// The VM will be created and started through startSandbox().
func (clh *cloudHypervisor) createSandbox(ctx context.Context, id string, networkNS NetworkNamespace, hypervisorConfig *HypervisorConfig) error {
@ -197,13 +208,11 @@ func (clh *cloudHypervisor) createSandbox(ctx context.Context, id string, networ
clh.ctx = newCtx
defer span.End()
err := hypervisorConfig.valid()
if err != nil {
if err := clh.setConfig(hypervisorConfig); err != nil {
return err
}
clh.id = id
clh.config = *hypervisorConfig
clh.state.state = clhNotReady
clh.Logger().WithField("function", "createSandbox").Info("creating Sandbox")

View File

@ -11,11 +11,13 @@ import (
"os"
"path/filepath"
"reflect"
"strings"
"testing"
"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist"
chclient "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/cloud-hypervisor/client"
"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils"
"github.com/pkg/errors"
"github.com/stretchr/testify/assert"
@ -366,3 +368,45 @@ func TestCloudHypervisorHotplugRemoveDevice(t *testing.T) {
_, err = clh.hotplugRemoveDevice(context.Background(), nil, netDev)
assert.Error(err, "Hotplug remove pmem block device expected error")
}
func TestClhGenerateSocket(t *testing.T) {
assert := assert.New(t)
// Ensure the type is fully constructed
hypervisor, err := NewHypervisor("clh")
assert.NoError(err)
clh, ok := hypervisor.(*cloudHypervisor)
assert.True(ok)
clh.addVSock(1, "path")
s, err := clh.generateSocket("c")
assert.NoError(err)
assert.NotNil(s)
hvsock, ok := s.(types.HybridVSock)
assert.True(ok)
assert.NotEmpty(hvsock.UdsPath)
// Path must be absolute
assert.True(strings.HasPrefix(hvsock.UdsPath, "/"))
assert.NotZero(hvsock.Port)
}
func TestClhSetConfig(t *testing.T) {
assert := assert.New(t)
config, err := newClhConfig()
assert.NoError(err)
clh := &cloudHypervisor{}
assert.Equal(clh.config, HypervisorConfig{})
err = clh.setConfig(&config)
assert.NoError(err)
assert.Equal(clh.config, config)
}

View File

@ -146,15 +146,16 @@ type firecracker struct {
fcConfig *types.FcConfig // Parameters configured before VM starts
connection *client.Firecracker //Tracks the current active connection
id string //Unique ID per pod. Normally maps to the sandbox id
vmPath string //All jailed VM assets need to be under this
chrootBaseDir string //chroot base for the jailer
jailerRoot string
socketPath string
netNSPath string
uid string //UID and GID to be used for the VMM
gid string
fcConfigPath string
id string //Unique ID per pod. Normally maps to the sandbox id
vmPath string //All jailed VM assets need to be under this
chrootBaseDir string //chroot base for the jailer
jailerRoot string
socketPath string
hybridSocketPath string
netNSPath string
uid string //UID and GID to be used for the VMM
gid string
fcConfigPath string
info FirecrackerInfo
config HypervisorConfig
@ -186,6 +187,17 @@ func (fc *firecracker) truncateID(id string) string {
return id
}
func (fc *firecracker) setConfig(config *HypervisorConfig) error {
err := config.valid()
if err != nil {
return err
}
fc.config = *config
return nil
}
// For firecracker this call only sets the internal structure up.
// The sandbox will be created and started through startSandbox().
func (fc *firecracker) createSandbox(ctx context.Context, id string, networkNS NetworkNamespace, hypervisorConfig *HypervisorConfig) error {
@ -198,8 +210,27 @@ func (fc *firecracker) createSandbox(ctx context.Context, id string, networkNS N
//https://github.com/kata-containers/runtime/issues/1065
fc.id = fc.truncateID(id)
fc.state.set(notReady)
fc.config = *hypervisorConfig
if err := fc.setConfig(hypervisorConfig); err != nil {
return err
}
fc.setPaths(&fc.config)
// So we need to repopulate this at startSandbox where it is valid
fc.netNSPath = networkNS.NetNsPath
// Till we create lower privileged kata user run as root
// https://github.com/kata-containers/runtime/issues/1869
fc.uid = "0"
fc.gid = "0"
fc.fcConfig = &types.FcConfig{}
fc.fcConfigPath = filepath.Join(fc.vmPath, defaultFcConfig)
return nil
}
func (fc *firecracker) setPaths(hypervisorConfig *HypervisorConfig) {
// When running with jailer all resources need to be under
// a specific location and that location needs to have
// exec permission (i.e. should not be mounted noexec, e.g. /run, /var/run)
@ -220,17 +251,7 @@ func (fc *firecracker) createSandbox(ctx context.Context, id string, networkNS N
// with the name of "firecracker.socket"
fc.socketPath = filepath.Join(fc.jailerRoot, "run", fcSocket)
// So we need to repopulate this at startSandbox where it is valid
fc.netNSPath = networkNS.NetNsPath
// Till we create lower privileged kata user run as root
// https://github.com/kata-containers/runtime/issues/1869
fc.uid = "0"
fc.gid = "0"
fc.fcConfig = &types.FcConfig{}
fc.fcConfigPath = filepath.Join(fc.vmPath, defaultFcConfig)
return nil
fc.hybridSocketPath = filepath.Join(fc.jailerRoot, defaultHybridVSocketName)
}
func (fc *firecracker) newFireClient(ctx context.Context) *client.Firecracker {
@ -783,7 +804,7 @@ func (fc *firecracker) startSandbox(ctx context.Context, timeout int) error {
}
// make sure 'others' don't have access to this socket
err = os.Chmod(filepath.Join(fc.jailerRoot, defaultHybridVSocketName), 0640)
err = os.Chmod(fc.hybridSocketPath, 0640)
if err != nil {
return fmt.Errorf("Could not change socket permissions: %v", err)
}
@ -1225,10 +1246,15 @@ func (fc *firecracker) check() error {
func (fc *firecracker) generateSocket(id string) (interface{}, error) {
fc.Logger().Debug("Using hybrid-vsock endpoint")
udsPath := filepath.Join(fc.jailerRoot, defaultHybridVSocketName)
// Method is being run outside of the normal container workflow
if fc.jailerRoot == "" {
fc.id = id
fc.setPaths(&fc.config)
}
return types.HybridVSock{
UdsPath: udsPath,
UdsPath: fc.hybridSocketPath,
Port: uint32(vSockPort),
}, nil
}

View File

@ -6,6 +6,7 @@
package virtcontainers
import (
"strings"
"testing"
"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
@ -23,6 +24,10 @@ func TestFCGenerateSocket(t *testing.T) {
hvsock, ok := i.(types.HybridVSock)
assert.True(ok)
assert.NotEmpty(hvsock.UdsPath)
// Path must be absolute
assert.True(strings.HasPrefix(hvsock.UdsPath, "/"))
assert.NotZero(hvsock.Port)
}
@ -64,3 +69,24 @@ func TestFCParseVersion(t *testing.T) {
assert.Equal(parsedVersion, v)
}
}
func TestFcSetConfig(t *testing.T) {
assert := assert.New(t)
config := HypervisorConfig{
HypervisorPath: "/some/where/firecracker",
KernelPath: "/some/where/kernel",
ImagePath: "/some/where/image",
JailerPath: "/some/where/jailer",
Debug: true,
}
fc := firecracker{}
assert.Equal(fc.config, HypervisorConfig{})
err := fc.setConfig(&config)
assert.NoError(err)
assert.Equal(fc.config, config)
}

View File

@ -185,8 +185,8 @@ func (hType *HypervisorType) String() string {
}
}
// newHypervisor returns an hypervisor from and hypervisor type.
func newHypervisor(hType HypervisorType) (hypervisor, error) {
// NewHypervisor returns an hypervisor from and hypervisor type.
func NewHypervisor(hType HypervisorType) (hypervisor, error) {
store, err := persist.GetDriver()
if err != nil {
return nil, err
@ -214,6 +214,41 @@ func newHypervisor(hType HypervisorType) (hypervisor, error) {
}
}
// GetHypervisorSocketTemplate returns the full "template" path to the
// hypervisor socket. If the specified hypervisor doesn't use a socket,
// an empty string is returned.
//
// The returned value is not the actual socket path since this function
// does not create a sandbox. Instead a path is returned with a special
// template value "{ID}" which would be replaced with the real sandbox
// name sandbox creation time.
func GetHypervisorSocketTemplate(hType HypervisorType, config *HypervisorConfig) (string, error) {
hypervisor, err := NewHypervisor(hType)
if err != nil {
return "", err
}
if err := hypervisor.setConfig(config); err != nil {
return "", err
}
// Tag that is used to represent the name of a sandbox
const sandboxID = "{ID}"
socket, err := hypervisor.generateSocket(sandboxID)
if err != nil {
return "", err
}
var socketPath string
if hybridVsock, ok := socket.(types.HybridVSock); ok {
socketPath = hybridVsock.UdsPath
}
return socketPath, nil
}
// Param is a key/value representation for hypervisor and kernel parameters.
type Param struct {
Key string
@ -858,6 +893,7 @@ func generateVMSocket(id string, vmStogarePath string) (interface{}, error) {
// hypervisor is the virtcontainers hypervisor interface.
// The default hypervisor implementation is Qemu.
type hypervisor interface {
setConfig(config *HypervisorConfig) error
createSandbox(ctx context.Context, id string, networkNS NetworkNamespace, hypervisorConfig *HypervisorConfig) error
startSandbox(ctx context.Context, timeout int) error
// If wait is set, don't actively stop the sandbox:

View File

@ -67,7 +67,7 @@ func TestStringFromUnknownHypervisorType(t *testing.T) {
func testNewHypervisorFromHypervisorType(t *testing.T, hypervisorType HypervisorType, expected hypervisor) {
assert := assert.New(t)
hy, err := newHypervisor(hypervisorType)
hy, err := NewHypervisor(hypervisorType)
assert.NoError(err)
assert.Exactly(hy, expected)
}
@ -82,7 +82,7 @@ func TestNewHypervisorFromUnknownHypervisorType(t *testing.T) {
var hypervisorType HypervisorType
assert := assert.New(t)
hy, err := newHypervisor(hypervisorType)
hy, err := NewHypervisor(hypervisorType)
assert.Error(err)
assert.Nil(hy)
}

View File

@ -30,9 +30,16 @@ func (m *mockHypervisor) hypervisorConfig() HypervisorConfig {
return HypervisorConfig{}
}
func (m *mockHypervisor) setConfig(config *HypervisorConfig) error {
if err := config.valid(); err != nil {
return err
}
return nil
}
func (m *mockHypervisor) createSandbox(ctx context.Context, id string, networkNS NetworkNamespace, hypervisorConfig *HypervisorConfig) error {
err := hypervisorConfig.valid()
if err != nil {
if err := m.setConfig(hypervisorConfig); err != nil {
return err
}

View File

@ -229,13 +229,14 @@ func (q *qemu) setup(ctx context.Context, id string, hypervisorConfig *Hyperviso
span, _ := katatrace.Trace(ctx, q.Logger(), "setup", qemuTracingTags, map[string]string{"sandbox_id": q.id})
defer span.End()
err := hypervisorConfig.valid()
if err != nil {
if err := q.setConfig(hypervisorConfig); err != nil {
return err
}
q.id = id
q.config = *hypervisorConfig
var err error
q.arch, err = newQemuArch(q.config)
if err != nil {
return err
@ -464,6 +465,17 @@ func (q *qemu) setupFileBackedMem(knobs *govmmQemu.Knobs, memory *govmmQemu.Memo
memory.Path = target
}
func (q *qemu) setConfig(config *HypervisorConfig) error {
err := config.valid()
if err != nil {
return err
}
q.config = *config
return nil
}
// createSandbox is the Hypervisor sandbox creation implementation for govmmQemu.
func (q *qemu) createSandbox(ctx context.Context, id string, networkNS NetworkNamespace, hypervisorConfig *HypervisorConfig) error {
// Save the tracing context

View File

@ -585,3 +585,17 @@ func TestQemuGetpids(t *testing.T) {
assert.True(pids[0] == 100)
assert.True(pids[1] == 200)
}
func TestQemuSetConfig(t *testing.T) {
assert := assert.New(t)
config := newQemuConfig()
q := &qemu{}
assert.Equal(q.config, HypervisorConfig{})
err := q.setConfig(&config)
assert.NoError(err)
assert.Equal(q.config, config)
}

View File

@ -152,7 +152,7 @@ func (sandboxConfig *SandboxConfig) valid() bool {
return false
}
if _, err := newHypervisor(sandboxConfig.HypervisorType); err != nil {
if _, err := NewHypervisor(sandboxConfig.HypervisorType); err != nil {
sandboxConfig.HypervisorType = QemuHypervisor
}
@ -498,7 +498,7 @@ func newSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Factor
// create agent instance
agent := getNewAgentFunc(ctx)()
hypervisor, err := newHypervisor(sandboxConfig.HypervisorType)
hypervisor, err := NewHypervisor(sandboxConfig.HypervisorType)
if err != nil {
return nil, err
}

View File

@ -85,7 +85,7 @@ func GrpcToVMConfig(j *pb.GrpcVMConfig) (*VMConfig, error) {
// NewVM creates a new VM based on provided VMConfig.
func NewVM(ctx context.Context, config VMConfig) (*VM, error) {
// 1. setup hypervisor
hypervisor, err := newHypervisor(config.HypervisorType)
hypervisor, err := NewHypervisor(config.HypervisorType)
if err != nil {
return nil, err
}
@ -165,7 +165,7 @@ func NewVM(ctx context.Context, config VMConfig) (*VM, error) {
func NewVMFromGrpc(ctx context.Context, v *pb.GrpcVM, config VMConfig) (*VM, error) {
virtLog.WithField("GrpcVM", v).WithField("config", config).Info("create new vm from Grpc")
hypervisor, err := newHypervisor(config.HypervisorType)
hypervisor, err := NewHypervisor(config.HypervisorType)
if err != nil {
return nil, err
}