From a0e0e18639e6a4bbaa347bf6360b5940db350ecb Mon Sep 17 00:00:00 2001 From: Eric Ernst Date: Thu, 11 Nov 2021 16:12:24 -0800 Subject: [PATCH 1/6] hypervisors: introduce pkg to unbreak vc/persist dependency Initial hypervisors pkg, with just basic state types defined. Fixes: #2883 Signed-off-by: Eric Ernst --- .../pkg/hypervisors/hypervisor_state.go | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 src/runtime/pkg/hypervisors/hypervisor_state.go diff --git a/src/runtime/pkg/hypervisors/hypervisor_state.go b/src/runtime/pkg/hypervisors/hypervisor_state.go new file mode 100644 index 0000000000..6dae6222ca --- /dev/null +++ b/src/runtime/pkg/hypervisors/hypervisor_state.go @@ -0,0 +1,50 @@ +// Copyright (c) 2019 Huawei Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package hypervisors + +// Bridge is a bridge where devices can be hot plugged +type Bridge struct { + // DeviceAddr contains information about devices plugged and its address in the bridge + DeviceAddr map[uint32]string + + // Type is the type of the bridge (pci, pcie, etc) + Type string + + //ID is used to identify the bridge in the hypervisor + ID string + + // Addr is the PCI/e slot of the bridge + Addr int +} + +// CPUDevice represents a CPU device which was hot-added in a running VM +type CPUDevice struct { + // ID is used to identify this CPU in the hypervisor options. + ID string +} + +type HypervisorState struct { + BlockIndexMap map[int]struct{} + + // Type of hypervisor, E.g. qemu/firecracker/acrn. + Type string + UUID string + // clh sepcific: refer to 'virtcontainers/clh.go:CloudHypervisorState' + APISocket string + + // Belows are qemu specific + // Refs: virtcontainers/qemu.go:QemuState + Bridges []Bridge + // HotpluggedCPUs is the list of CPUs that were hot-added + HotpluggedVCPUs []CPUDevice + + HotpluggedMemory int + VirtiofsdPid int + Pid int + PCIeRootPort int + + HotplugVFIOOnRootBus bool +} From c28e5a780782c473f3b8f86a29c1a729d1e3d5b4 Mon Sep 17 00:00:00 2001 From: Eric Ernst Date: Wed, 17 Nov 2021 15:38:26 -0800 Subject: [PATCH 2/6] acrn: remove dependency on sandbox, persistapi datatypes Today, acrn relies on sandbox level information, as well as a store provided by common parts of the hypervisor. As we cleanup the abstractions within our runtime, we need to ensure that there aren't cross dependencies between the sandbox, the persistence logic and the hypervisor. Ensure that ACRN still compiles, but remove the setSandbox usage as well as persist driver setup. Signed-off-by: Eric Ernst --- src/runtime/virtcontainers/acrn.go | 54 +++++++++++++++--------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/src/runtime/virtcontainers/acrn.go b/src/runtime/virtcontainers/acrn.go index fafc734a56..db0f6491aa 100644 --- a/src/runtime/virtcontainers/acrn.go +++ b/src/runtime/virtcontainers/acrn.go @@ -7,7 +7,6 @@ package virtcontainers import ( "context" - "encoding/json" "fmt" "os" "os/exec" @@ -20,6 +19,7 @@ import ( "github.com/pkg/errors" "github.com/sirupsen/logrus" + hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors" "github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace" "github.com/kata-containers/kata-containers/src/runtime/pkg/uuid" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config" @@ -39,11 +39,13 @@ var acrnTracingTags = map[string]string{ // Since ACRN is using the store in a quite abnormal way, let's first draw it back from store to here +/* // UUIDPathSuffix is the suffix used for uuid storage const ( UUIDPathSuffix = "uuid" uuidFile = "uuid.json" ) +*/ // ACRN currently supports only known UUIDs for security // reasons (FuSa). When launching VM, only these pre-defined @@ -312,7 +314,7 @@ func (a *Acrn) setup(ctx context.Context, id string, hypervisorConfig *Hyperviso // The path might already exist, but in case of VM templating, // we have to create it since the sandbox has not created it yet. - if err = os.MkdirAll(filepath.Join(a.store.RunStoragePath(), id), DirMode); err != nil { + if err = os.MkdirAll(filepath.Join(a.config.RunStorePath, id), DirMode); err != nil { return err } @@ -438,7 +440,7 @@ func (a *Acrn) StartVM(ctx context.Context, timeoutSecs int) error { a.Logger().WithField("default-kernel-parameters", formatted).Debug() } - vmPath := filepath.Join(a.store.RunVMStoragePath(), a.id) + vmPath := filepath.Join(a.config.VMStorePath, a.id) err := os.MkdirAll(vmPath, DirMode) if err != nil { return err @@ -634,7 +636,7 @@ func (a *Acrn) GetVMConsole(ctx context.Context, id string) (string, string, err span, _ := katatrace.Trace(ctx, a.Logger(), "GetVMConsole", acrnTracingTags, map[string]string{"sandbox_id": a.id}) defer span.End() - consoleURL, err := utils.BuildSocketPath(a.store.RunVMStoragePath(), id, acrnConsoleSocket) + consoleURL, err := utils.BuildSocketPath(a.config.VMStorePath, id, acrnConsoleSocket) if err != nil { return consoleProtoUnix, "", err } @@ -698,14 +700,14 @@ func (a *Acrn) toGrpc(ctx context.Context) ([]byte, error) { return nil, errors.New("acrn is not supported by VM cache") } -func (a *Acrn) Save() (s persistapi.HypervisorState) { +func (a *Acrn) Save() (s hv.HypervisorState) { s.Pid = a.state.PID s.Type = string(AcrnHypervisor) s.UUID = a.state.UUID return } -func (a *Acrn) Load(s persistapi.HypervisorState) { +func (a *Acrn) Load(s hv.HypervisorState) { a.state.PID = s.Pid a.state.UUID = s.UUID } @@ -719,7 +721,7 @@ func (a *Acrn) Check() error { } func (a *Acrn) GenerateSocket(id string) (interface{}, error) { - return generateVMSocket(id, a.store.RunVMStoragePath()) + return generateVMSocket(id, a.config.VMStorePath) } // GetACRNUUIDBytes returns UUID bytes that is used for VM creation @@ -782,38 +784,36 @@ func (a *Acrn) GetMaxSupportedACRNVM() (uint8, error) { } func (a *Acrn) storeInfo() error { - relPath := filepath.Join(UUIDPathSuffix, uuidFile) + /* + relPath := filepath.Join(UUIDPathSuffix, uuidFile) - jsonOut, err := json.Marshal(a.info) - if err != nil { - return fmt.Errorf("Could not marshal data: %s", err) - } + jsonOut, err := json.Marshal(a.info) + if err != nil { + return fmt.Errorf("Could not marshal data: %s", err) + } - if err := a.store.GlobalWrite(relPath, jsonOut); err != nil { - return fmt.Errorf("failed to write uuid to file: %v", err) - } + if err := a.store.GlobalWrite(relPath, jsonOut); err != nil { + return fmt.Errorf("failed to write uuid to file: %v", err) + }*/ return nil } func (a *Acrn) loadInfo() error { - relPath := filepath.Join(UUIDPathSuffix, uuidFile) + /* + relPath := filepath.Join(UUIDPathSuffix, uuidFile) + data, err := a.store.GlobalRead(relPath) + if err != nil { + return fmt.Errorf("failed to read uuid from file: %v", err) + } - data, err := a.store.GlobalRead(relPath) - if err != nil { - return fmt.Errorf("failed to read uuid from file: %v", err) - } + if err := json.Unmarshal(data, &a.info); err != nil { + return fmt.Errorf("failed to unmarshal uuid info: %v", err) + }*/ - if err := json.Unmarshal(data, &a.info); err != nil { - return fmt.Errorf("failed to unmarshal uuid info: %v", err) - } return nil } func (a *Acrn) IsRateLimiterBuiltin() bool { return false } - -func (a *Acrn) setSandbox(sandbox *Sandbox) { - a.sandbox = sandbox -} From 34f23de512f4499b7e8edf46159dc16ce19d2196 Mon Sep 17 00:00:00 2001 From: Eric Ernst Date: Tue, 9 Nov 2021 11:31:44 -0800 Subject: [PATCH 3/6] vc: hypervisor: Remove need to get shared address from sandbox Add shared path as part of the hypervisor config Signed-off-by: Eric Ernst --- src/runtime/pkg/katautils/create.go | 3 +++ src/runtime/virtcontainers/clh.go | 2 +- src/runtime/virtcontainers/hypervisor.go | 13 ++++++++----- src/runtime/virtcontainers/kata_agent.go | 2 +- src/runtime/virtcontainers/qemu.go | 2 +- 5 files changed, 14 insertions(+), 8 deletions(-) diff --git a/src/runtime/pkg/katautils/create.go b/src/runtime/pkg/katautils/create.go index fc593c6712..dd7056214a 100644 --- a/src/runtime/pkg/katautils/create.go +++ b/src/runtime/pkg/katautils/create.go @@ -120,6 +120,9 @@ func CreateSandbox(ctx context.Context, vci vc.VC, ociSpec specs.Spec, runtimeCo return nil, vc.Process{}, err } + // setup shared path in hypervisor config: + sandboxConfig.HypervisorConfig.SharedPath = vc.GetSharePath(containerID) + if err := checkForFIPS(&sandboxConfig); err != nil { return nil, vc.Process{}, err } diff --git a/src/runtime/virtcontainers/clh.go b/src/runtime/virtcontainers/clh.go index 9236c832cd..315cb599f5 100644 --- a/src/runtime/virtcontainers/clh.go +++ b/src/runtime/virtcontainers/clh.go @@ -226,7 +226,7 @@ func (clh *cloudHypervisor) CreateVM(ctx context.Context, id string, networkNS N clh.Logger().WithField("function", "CreateVM").Info("Sandbox already exist, loading from state") clh.virtiofsd = &virtiofsd{ PID: clh.state.VirtiofsdPID, - sourcePath: filepath.Join(getSharePath(clh.id)), + sourcePath: hypervisorConfig.SharedPath, debug: clh.config.Debug, socketPath: virtiofsdSocketPath, } diff --git a/src/runtime/virtcontainers/hypervisor.go b/src/runtime/virtcontainers/hypervisor.go index 0186c397ed..bc22a17112 100644 --- a/src/runtime/virtcontainers/hypervisor.go +++ b/src/runtime/virtcontainers/hypervisor.go @@ -315,13 +315,19 @@ type HypervisorConfig struct { EntropySource string // Shared file system type: - // - virtio-9p (default) - // - virtio-fs + // - virtio-9p + // - virtio-fs (default) SharedFS string + // Path for filesystem sharing + SharedPath string + // VirtioFSDaemon is the virtio-fs vhost-user daemon path VirtioFSDaemon string + // VirtioFSCache cache mode for fs version cache or "none" + VirtioFSCache string + // File based memory backend root directory FileBackedMemRootDir string @@ -342,9 +348,6 @@ type HypervisorConfig struct { // SELinux label for the VM SELinuxProcessLabel string - // VirtioFSCache cache mode for fs version cache or "none" - VirtioFSCache string - // HypervisorPathList is the list of hypervisor paths names allowed in annotations HypervisorPathList []string diff --git a/src/runtime/virtcontainers/kata_agent.go b/src/runtime/virtcontainers/kata_agent.go index 3562c9216b..b604917b2c 100644 --- a/src/runtime/virtcontainers/kata_agent.go +++ b/src/runtime/virtcontainers/kata_agent.go @@ -162,7 +162,7 @@ var kataHostSharedDir = func() string { // 2. /run/kata-containers/shared/sandboxes/$sbx_id/mounts/ is bind mounted readonly to /run/kata-containers/shared/sandboxes/$sbx_id/shared/, so guest cannot modify it // // 3. host-guest shared files/directories are mounted one-level under /run/kata-containers/shared/sandboxes/$sbx_id/mounts/ and thus present to guest at one level under /run/kata-containers/shared/sandboxes/$sbx_id/shared/ -func getSharePath(id string) string { +func GetSharePath(id string) string { return filepath.Join(kataHostSharedDir(), id, "shared") } diff --git a/src/runtime/virtcontainers/qemu.go b/src/runtime/virtcontainers/qemu.go index 29cd0d1731..8db487a809 100644 --- a/src/runtime/virtcontainers/qemu.go +++ b/src/runtime/virtcontainers/qemu.go @@ -655,7 +655,7 @@ func (q *qemu) CreateVM(ctx context.Context, id string, networkNS NetworkNamespa q.virtiofsd = &virtiofsd{ path: q.config.VirtioFSDaemon, - sourcePath: filepath.Join(getSharePath(q.id)), + sourcePath: hypervisorConfig.SharedPath, socketPath: virtiofsdSocketPath, extraArgs: q.config.VirtioFSExtraArgs, debug: q.config.Debug, From 4c2883f7e25615a789ce232236b225e2571248ca Mon Sep 17 00:00:00 2001 From: Eric Ernst Date: Thu, 11 Nov 2021 15:26:10 -0800 Subject: [PATCH 4/6] vc: hypervisor: remove dependency on persist API Today the hypervisor code in vc relies on persist pkg for two things: 1. To get the VM/run store path on the host filesystem, 2. For type definition of the Load/Save functions of the hypervisor interface. For (1), we can simply remove the store interface from the hypervisor config and replace it with just the path, since this is all we really need. When we create a NewHypervisor structure, outside of the hypervisor, we can populate this path. For (2), rather than have the persist pkg define the structure, let's let the hypervisor code (soon to be pkg) define the structure. persist API already needs to call into hypervisor anyway; let's allow us to define the structure. We'll probably want to look at following similar pattern for other parts of vc that we want to make independent of the persist API. In doing this, we started an initial hypervisors pkg, to hold these types (avoid a circular dependency between virtcontainers and persist pkg). Next step will be to remove all other dependencies and move the hypervisor specific code into this pkg, and out of virtcontaienrs. Signed-off-by: Eric Ernst --- src/runtime/cmd/kata-runtime/kata-env_test.go | 4 +- src/runtime/virtcontainers/acrn_test.go | 12 ++++- src/runtime/virtcontainers/clh.go | 21 ++++---- src/runtime/virtcontainers/clh_test.go | 22 ++++++-- src/runtime/virtcontainers/fc.go | 6 +-- src/runtime/virtcontainers/hypervisor.go | 31 +++++------ src/runtime/virtcontainers/kata_agent.go | 8 +-- src/runtime/virtcontainers/kata_agent_test.go | 2 +- src/runtime/virtcontainers/mock_hypervisor.go | 6 +-- src/runtime/virtcontainers/persist.go | 3 +- .../virtcontainers/persist/api/hypervisor.go | 50 ----------------- .../virtcontainers/persist/api/sandbox.go | 6 ++- src/runtime/virtcontainers/qemu.go | 44 +++++++-------- src/runtime/virtcontainers/qemu_test.go | 53 ++++++++++++++----- src/runtime/virtcontainers/sandbox.go | 4 +- .../virtcontainers/virtcontainers_test.go | 2 +- 16 files changed, 133 insertions(+), 141 deletions(-) delete mode 100644 src/runtime/virtcontainers/persist/api/hypervisor.go diff --git a/src/runtime/cmd/kata-runtime/kata-env_test.go b/src/runtime/cmd/kata-runtime/kata-env_test.go index 507fd03253..793c3df420 100644 --- a/src/runtime/cmd/kata-runtime/kata-env_test.go +++ b/src/runtime/cmd/kata-runtime/kata-env_test.go @@ -1052,11 +1052,13 @@ func TestGetHypervisorInfoSocket(t *testing.T) { {vc.QemuHypervisor, false}, } + config.HypervisorConfig.VMStorePath = "/foo" + config.HypervisorConfig.RunStorePath = "/bar" + for i, details := range hypervisors { msg := fmt.Sprintf("hypervisor[%d]: %+v", i, details) config.HypervisorType = details.hType - info, err := getHypervisorInfo(config) assert.NoError(err, msg) diff --git a/src/runtime/virtcontainers/acrn_test.go b/src/runtime/virtcontainers/acrn_test.go index f3d46b7b8b..bb19b45a7a 100644 --- a/src/runtime/virtcontainers/acrn_test.go +++ b/src/runtime/virtcontainers/acrn_test.go @@ -199,11 +199,15 @@ func TestAcrnGetSandboxConsole(t *testing.T) { assert.NoError(err) a := &Acrn{ - ctx: context.Background(), + ctx: context.Background(), + config: HypervisorConfig{ + VMStorePath: store.RunVMStoragePath(), + RunStorePath: store.RunStoragePath(), + }, store: store, } sandboxID := "testSandboxID" - expected := filepath.Join(a.store.RunVMStoragePath(), sandboxID, consoleSocket) + expected := filepath.Join(store.RunVMStoragePath(), sandboxID, consoleSocket) proto, result, err := a.GetVMConsole(a.ctx, sandboxID) assert.NoError(err) @@ -219,6 +223,10 @@ func TestAcrnCreateVM(t *testing.T) { a := &Acrn{ store: store, + config: HypervisorConfig{ + VMStorePath: store.RunVMStoragePath(), + RunStorePath: store.RunStoragePath(), + }, } sandbox := &Sandbox{ diff --git a/src/runtime/virtcontainers/clh.go b/src/runtime/virtcontainers/clh.go index 315cb599f5..e4ca0a0061 100644 --- a/src/runtime/virtcontainers/clh.go +++ b/src/runtime/virtcontainers/clh.go @@ -20,12 +20,12 @@ import ( "time" "github.com/containerd/console" - persistapi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/api" chclient "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/cloud-hypervisor/client" "github.com/opencontainers/selinux/go-selinux/label" "github.com/pkg/errors" log "github.com/sirupsen/logrus" + hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors" "github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types" @@ -157,7 +157,6 @@ func (s *CloudHypervisorState) reset() { } type cloudHypervisor struct { - store persistapi.PersistDriver console console.Console virtiofsd Virtiofsd APIClient clhClient @@ -342,7 +341,7 @@ func (clh *cloudHypervisor) CreateVM(ctx context.Context, id string, networkNS N clh.virtiofsd = &virtiofsd{ path: clh.config.VirtioFSDaemon, - sourcePath: filepath.Join(getSharePath(clh.id)), + sourcePath: filepath.Join(GetSharePath(clh.id)), socketPath: virtiofsdSocketPath, extraArgs: clh.config.VirtioFSExtraArgs, debug: clh.config.Debug, @@ -374,7 +373,7 @@ func (clh *cloudHypervisor) StartVM(ctx context.Context, timeout int) error { clh.Logger().WithField("function", "StartVM").Info("starting Sandbox") - vmPath := filepath.Join(clh.store.RunVMStoragePath(), clh.id) + vmPath := filepath.Join(clh.config.VMStorePath, clh.id) err := os.MkdirAll(vmPath, DirMode) if err != nil { return err @@ -747,7 +746,7 @@ func (clh *cloudHypervisor) toGrpc(ctx context.Context) ([]byte, error) { return nil, errors.New("cloudHypervisor is not supported by VM cache") } -func (clh *cloudHypervisor) Save() (s persistapi.HypervisorState) { +func (clh *cloudHypervisor) Save() (s hv.HypervisorState) { s.Pid = clh.state.PID s.Type = string(ClhHypervisor) s.VirtiofsdPid = clh.state.VirtiofsdPID @@ -755,7 +754,7 @@ func (clh *cloudHypervisor) Save() (s persistapi.HypervisorState) { return } -func (clh *cloudHypervisor) Load(s persistapi.HypervisorState) { +func (clh *cloudHypervisor) Load(s hv.HypervisorState) { clh.state.PID = s.Pid clh.state.VirtiofsdPID = s.VirtiofsdPid clh.state.apiSocket = s.APISocket @@ -893,15 +892,15 @@ func (clh *cloudHypervisor) GenerateSocket(id string) (interface{}, error) { } func (clh *cloudHypervisor) virtioFsSocketPath(id string) (string, error) { - return utils.BuildSocketPath(clh.store.RunVMStoragePath(), id, virtioFsSocket) + return utils.BuildSocketPath(clh.config.VMStorePath, id, virtioFsSocket) } func (clh *cloudHypervisor) vsockSocketPath(id string) (string, error) { - return utils.BuildSocketPath(clh.store.RunVMStoragePath(), id, clhSocket) + return utils.BuildSocketPath(clh.config.VMStorePath, id, clhSocket) } func (clh *cloudHypervisor) apiSocketPath(id string) (string, error) { - return utils.BuildSocketPath(clh.store.RunVMStoragePath(), id, clhAPISocket) + return utils.BuildSocketPath(clh.config.VMStorePath, id, clhAPISocket) } func (clh *cloudHypervisor) waitVMM(timeout uint) error { @@ -1213,7 +1212,7 @@ func (clh *cloudHypervisor) cleanupVM(force bool) error { } // Cleanup vm path - dir := filepath.Join(clh.store.RunVMStoragePath(), clh.id) + dir := filepath.Join(clh.config.VMStorePath, clh.id) // If it's a symlink, remove both dir and the target. link, err := filepath.EvalSymlinks(dir) @@ -1242,7 +1241,7 @@ func (clh *cloudHypervisor) cleanupVM(force bool) error { } if clh.config.VMid != "" { - dir = filepath.Join(clh.store.RunStoragePath(), clh.config.VMid) + dir = filepath.Join(clh.config.VMStorePath, clh.config.VMid) if err := os.RemoveAll(dir); err != nil { if !force { return err diff --git a/src/runtime/virtcontainers/clh_test.go b/src/runtime/virtcontainers/clh_test.go index 54718dd57c..b0de2ae9be 100644 --- a/src/runtime/virtcontainers/clh_test.go +++ b/src/runtime/virtcontainers/clh_test.go @@ -203,7 +203,10 @@ func TestCloudHypervisorCleanupVM(t *testing.T) { assert.NoError(err, "persist.GetDriver() unexpected error") clh := &cloudHypervisor{ - store: store, + config: HypervisorConfig{ + VMStorePath: store.RunVMStoragePath(), + RunStorePath: store.RunStoragePath(), + }, } err = clh.cleanupVM(true) @@ -214,7 +217,7 @@ func TestCloudHypervisorCleanupVM(t *testing.T) { err = clh.cleanupVM(true) assert.NoError(err, "persist.GetDriver() unexpected error") - dir := filepath.Join(clh.store.RunVMStoragePath(), clh.id) + dir := filepath.Join(store.RunVMStoragePath(), clh.id) os.MkdirAll(dir, os.ModePerm) err = clh.cleanupVM(false) @@ -235,9 +238,11 @@ func TestClhCreateVM(t *testing.T) { store, err := persist.GetDriver() assert.NoError(err) + clhConfig.VMStorePath = store.RunVMStoragePath() + clhConfig.RunStorePath = store.RunStoragePath() + clh := &cloudHypervisor{ config: clhConfig, - store: store, } sandbox := &Sandbox{ @@ -261,11 +266,13 @@ func TestClooudHypervisorStartSandbox(t *testing.T) { store, err := persist.GetDriver() assert.NoError(err) + clhConfig.VMStorePath = store.RunVMStoragePath() + clhConfig.RunStorePath = store.RunStoragePath() + clh := &cloudHypervisor{ config: clhConfig, APIClient: &clhClientMock{}, virtiofsd: &virtiofsdMock{}, - store: store, } err = clh.StartVM(context.Background(), 10) @@ -379,6 +386,11 @@ func TestClhGenerateSocket(t *testing.T) { clh, ok := hypervisor.(*cloudHypervisor) assert.True(ok) + clh.config = HypervisorConfig{ + VMStorePath: "/foo", + RunStorePath: "/bar", + } + clh.addVSock(1, "path") s, err := clh.GenerateSocket("c") @@ -391,7 +403,7 @@ func TestClhGenerateSocket(t *testing.T) { assert.NotEmpty(hvsock.UdsPath) // Path must be absolute - assert.True(strings.HasPrefix(hvsock.UdsPath, "/")) + assert.True(strings.HasPrefix(hvsock.UdsPath, "/"), "failed: socket path: %s", hvsock.UdsPath) assert.NotZero(hvsock.Port) } diff --git a/src/runtime/virtcontainers/fc.go b/src/runtime/virtcontainers/fc.go index 1b4c055ccf..fa4e56ffab 100644 --- a/src/runtime/virtcontainers/fc.go +++ b/src/runtime/virtcontainers/fc.go @@ -22,9 +22,9 @@ import ( "syscall" "time" + hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors" "github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config" - persistapi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/api" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/firecracker/client" models "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/firecracker/client/models" ops "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/firecracker/client/operations" @@ -1226,13 +1226,13 @@ func (fc *firecracker) toGrpc(ctx context.Context) ([]byte, error) { return nil, errors.New("firecracker is not supported by VM cache") } -func (fc *firecracker) Save() (s persistapi.HypervisorState) { +func (fc *firecracker) Save() (s hv.HypervisorState) { s.Pid = fc.info.PID s.Type = string(FirecrackerHypervisor) return } -func (fc *firecracker) Load(s persistapi.HypervisorState) { +func (fc *firecracker) Load(s hv.HypervisorState) { fc.info.PID = s.Pid } diff --git a/src/runtime/virtcontainers/hypervisor.go b/src/runtime/virtcontainers/hypervisor.go index bc22a17112..f446a078bf 100644 --- a/src/runtime/virtcontainers/hypervisor.go +++ b/src/runtime/virtcontainers/hypervisor.go @@ -14,9 +14,8 @@ import ( "strconv" "strings" + hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist" - persistapi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/api" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils" ) @@ -185,28 +184,18 @@ func (hType *HypervisorType) String() string { } } -// NewHypervisor returns an hypervisor from and hypervisor type. +// NewHypervisor returns an hypervisor from a hypervisor type. func NewHypervisor(hType HypervisorType) (Hypervisor, error) { - store, err := persist.GetDriver() - if err != nil { - return nil, err - } switch hType { case QemuHypervisor: - return &qemu{ - store: store, - }, nil + return &qemu{}, nil case FirecrackerHypervisor: return &firecracker{}, nil case AcrnHypervisor: - return &Acrn{ - store: store, - }, nil + return &Acrn{}, nil case ClhHypervisor: - return &cloudHypervisor{ - store: store, - }, nil + return &cloudHypervisor{}, nil case MockHypervisor: return &mockHypervisor{}, nil default: @@ -345,6 +334,12 @@ type HypervisorConfig struct { // VMid is "" if the hypervisor is not created by the factory. VMid string + // VMStorePath is the location on disk where VM information will persist + VMStorePath string + + // VMStorePath is the location on disk where runtime information will persist + RunStorePath string + // SELinux label for the VM SELinuxProcessLabel string @@ -934,8 +929,8 @@ type Hypervisor interface { toGrpc(ctx context.Context) ([]byte, error) Check() error - Save() persistapi.HypervisorState - Load(persistapi.HypervisorState) + Save() hv.HypervisorState + Load(hv.HypervisorState) // generate the socket to communicate the host and guest GenerateSocket(id string) (interface{}, error) diff --git a/src/runtime/virtcontainers/kata_agent.go b/src/runtime/virtcontainers/kata_agent.go index b604917b2c..6cecbd4523 100644 --- a/src/runtime/virtcontainers/kata_agent.go +++ b/src/runtime/virtcontainers/kata_agent.go @@ -356,7 +356,7 @@ func (k *kataAgent) setupSandboxBindMounts(ctx context.Context, sandbox *Sandbox // Create subdirectory in host shared path for sandbox mounts sandboxMountDir := filepath.Join(getMountPath(sandbox.id), sandboxMountsDir) - sandboxShareDir := filepath.Join(getSharePath(sandbox.id), sandboxMountsDir) + sandboxShareDir := filepath.Join(GetSharePath(sandbox.id), sandboxMountsDir) if err := os.MkdirAll(sandboxMountDir, DirMode); err != nil { return fmt.Errorf("Creating sandbox shared mount directory: %v: %w", sandboxMountDir, err) } @@ -473,7 +473,7 @@ func (k *kataAgent) setupSharedPath(ctx context.Context, sandbox *Sandbox) (err defer span.End() // create shared path structure - sharePath := getSharePath(sandbox.id) + sharePath := GetSharePath(sandbox.id) mountPath := getMountPath(sandbox.id) if err := os.MkdirAll(sharePath, sharedDirMode); err != nil { return err @@ -509,7 +509,7 @@ func (k *kataAgent) createSandbox(ctx context.Context, sandbox *Sandbox) error { if err := k.setupSharedPath(ctx, sandbox); err != nil { return err } - return k.configure(ctx, sandbox.hypervisor, sandbox.id, getSharePath(sandbox.id), sandbox.config.AgentConfig) + return k.configure(ctx, sandbox.hypervisor, sandbox.id, GetSharePath(sandbox.id), sandbox.config.AgentConfig) } func cmdToKataProcess(cmd types.Cmd) (process *grpc.Process, err error) { @@ -2198,7 +2198,7 @@ func (k *kataAgent) cleanup(ctx context.Context, s *Sandbox) { } // Unmount shared path - path := getSharePath(s.id) + path := GetSharePath(s.id) k.Logger().WithField("path", path).Infof("Cleanup agent") if err := syscall.Unmount(path, syscall.MNT_DETACH|UmountNoFollow); err != nil { k.Logger().WithError(err).Errorf("failed to unmount vm share path %s", path) diff --git a/src/runtime/virtcontainers/kata_agent_test.go b/src/runtime/virtcontainers/kata_agent_test.go index 6e329919ed..48c0f6a73f 100644 --- a/src/runtime/virtcontainers/kata_agent_test.go +++ b/src/runtime/virtcontainers/kata_agent_test.go @@ -1158,7 +1158,7 @@ func TestSandboxBindMount(t *testing.T) { assert.Nil(err) defer os.RemoveAll(dir) - sharePath := getSharePath(sandbox.id) + sharePath := GetSharePath(sandbox.id) mountPath := getMountPath(sandbox.id) err = os.MkdirAll(sharePath, DirMode) diff --git a/src/runtime/virtcontainers/mock_hypervisor.go b/src/runtime/virtcontainers/mock_hypervisor.go index 2c132f0ba8..dc4b11ad67 100644 --- a/src/runtime/virtcontainers/mock_hypervisor.go +++ b/src/runtime/virtcontainers/mock_hypervisor.go @@ -10,7 +10,7 @@ import ( "errors" "os" - persistapi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/api" + hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types" ) @@ -130,11 +130,11 @@ func (m *mockHypervisor) toGrpc(ctx context.Context) ([]byte, error) { return nil, errors.New("mockHypervisor is not supported by VM cache") } -func (m *mockHypervisor) Save() (s persistapi.HypervisorState) { +func (m *mockHypervisor) Save() (s hv.HypervisorState) { return } -func (m *mockHypervisor) Load(s persistapi.HypervisorState) {} +func (m *mockHypervisor) Load(s hv.HypervisorState) {} func (m *mockHypervisor) Check() error { return nil diff --git a/src/runtime/virtcontainers/persist.go b/src/runtime/virtcontainers/persist.go index b8fc2c8abb..3fe45351dd 100644 --- a/src/runtime/virtcontainers/persist.go +++ b/src/runtime/virtcontainers/persist.go @@ -8,6 +8,7 @@ package virtcontainers import ( "errors" + hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/api" exp "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/experimental" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist" @@ -315,7 +316,7 @@ func (c *Container) loadContState(cs persistapi.ContainerState) { } } -func (s *Sandbox) loadHypervisor(hs persistapi.HypervisorState) { +func (s *Sandbox) loadHypervisor(hs hv.HypervisorState) { s.hypervisor.Load(hs) } diff --git a/src/runtime/virtcontainers/persist/api/hypervisor.go b/src/runtime/virtcontainers/persist/api/hypervisor.go deleted file mode 100644 index b2d41000f5..0000000000 --- a/src/runtime/virtcontainers/persist/api/hypervisor.go +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright (c) 2019 Huawei Corporation -// -// SPDX-License-Identifier: Apache-2.0 -// - -package persistapi - -// Bridge is a bridge where devices can be hot plugged -type Bridge struct { - // DeviceAddr contains information about devices plugged and its address in the bridge - DeviceAddr map[uint32]string - - // Type is the type of the bridge (pci, pcie, etc) - Type string - - //ID is used to identify the bridge in the hypervisor - ID string - - // Addr is the PCI/e slot of the bridge - Addr int -} - -// CPUDevice represents a CPU device which was hot-added in a running VM -type CPUDevice struct { - // ID is used to identify this CPU in the hypervisor options. - ID string -} - -type HypervisorState struct { - BlockIndexMap map[int]struct{} - - // Type of hypervisor, E.g. qemu/firecracker/acrn. - Type string - UUID string - // clh sepcific: refer to 'virtcontainers/clh.go:CloudHypervisorState' - APISocket string - - // Belows are qemu specific - // Refs: virtcontainers/qemu.go:QemuState - Bridges []Bridge - // HotpluggedCPUs is the list of CPUs that were hot-added - HotpluggedVCPUs []CPUDevice - - HotpluggedMemory int - VirtiofsdPid int - Pid int - PCIeRootPort int - - HotplugVFIOOnRootBus bool -} diff --git a/src/runtime/virtcontainers/persist/api/sandbox.go b/src/runtime/virtcontainers/persist/api/sandbox.go index 1398cc20f4..09196637ca 100644 --- a/src/runtime/virtcontainers/persist/api/sandbox.go +++ b/src/runtime/virtcontainers/persist/api/sandbox.go @@ -6,6 +6,10 @@ package persistapi +import ( + hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors" +) + // ============= sandbox level resources ============= // AgentState save agent state data @@ -38,7 +42,7 @@ type SandboxState struct { OverheadCgroupPath string // HypervisorState saves hypervisor specific data - HypervisorState HypervisorState + HypervisorState hv.HypervisorState // AgentState saves state data of agent AgentState AgentState diff --git a/src/runtime/virtcontainers/qemu.go b/src/runtime/virtcontainers/qemu.go index 8db487a809..1c1073b24f 100644 --- a/src/runtime/virtcontainers/qemu.go +++ b/src/runtime/virtcontainers/qemu.go @@ -31,11 +31,11 @@ import ( "github.com/sirupsen/logrus" "golang.org/x/sys/unix" + hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors" "github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace" pkgUtils "github.com/kata-containers/kata-containers/src/runtime/pkg/utils" "github.com/kata-containers/kata-containers/src/runtime/pkg/uuid" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config" - persistapi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/api" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types" vcTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils" @@ -67,18 +67,12 @@ type qmpChannel struct { sync.Mutex } -// CPUDevice represents a CPU device which was hot-added in a running VM -type CPUDevice struct { - // ID is used to identify this CPU in the hypervisor options. - ID string -} - // QemuState keeps Qemu's state type QemuState struct { UUID string Bridges []types.Bridge // HotpluggedCPUs is the list of CPUs that were hot-added - HotpluggedVCPUs []CPUDevice + HotpluggedVCPUs []hv.CPUDevice HotpluggedMemory int VirtiofsdPid int PCIeRootPort int @@ -92,8 +86,6 @@ type qemu struct { virtiofsd Virtiofsd - store persistapi.PersistDriver - ctx context.Context // fds is a list of file descriptors inherited by QEMU process @@ -276,7 +268,7 @@ func (q *qemu) setup(ctx context.Context, id string, hypervisorConfig *Hyperviso // The path might already exist, but in case of VM templating, // we have to create it since the sandbox has not created it yet. - if err = utils.MkdirAllWithInheritedOwner(filepath.Join(q.store.RunStoragePath(), id), DirMode); err != nil { + if err = utils.MkdirAllWithInheritedOwner(filepath.Join(q.config.RunStorePath, id), DirMode); err != nil { return err } } @@ -331,7 +323,7 @@ func (q *qemu) memoryTopology() (govmmQemu.Memory, error) { } func (q *qemu) qmpSocketPath(id string) (string, error) { - return utils.BuildSocketPath(q.store.RunVMStoragePath(), id, qmpSocket) + return utils.BuildSocketPath(q.config.VMStorePath, id, qmpSocket) } func (q *qemu) getQemuMachine() (govmmQemu.Machine, error) { @@ -618,7 +610,7 @@ func (q *qemu) CreateVM(ctx context.Context, id string, networkNS NetworkNamespa GlobalParam: "kvm-pit.lost_tick_policy=discard", Bios: firmwarePath, PFlash: pflash, - PidFile: filepath.Join(q.store.RunVMStoragePath(), q.id, "pid"), + PidFile: filepath.Join(q.config.VMStorePath, q.id, "pid"), } qemuConfig.Devices, qemuConfig.Bios, err = q.arch.appendProtectionDevice(qemuConfig.Devices, firmwarePath) @@ -666,7 +658,7 @@ func (q *qemu) CreateVM(ctx context.Context, id string, networkNS NetworkNamespa } func (q *qemu) vhostFSSocketPath(id string) (string, error) { - return utils.BuildSocketPath(q.store.RunVMStoragePath(), id, vhostFSSocket) + return utils.BuildSocketPath(q.config.VMStorePath, id, vhostFSSocket) } func (q *qemu) setupVirtiofsd(ctx context.Context) (err error) { @@ -795,7 +787,7 @@ func (q *qemu) StartVM(ctx context.Context, timeout int) error { q.fds = []*os.File{} }() - vmPath := filepath.Join(q.store.RunVMStoragePath(), q.id) + vmPath := filepath.Join(q.config.VMStorePath, q.id) err := utils.MkdirAllWithInheritedOwner(vmPath, DirMode) if err != nil { return err @@ -1002,7 +994,7 @@ func (q *qemu) StopVM(ctx context.Context, waitOnly bool) error { func (q *qemu) cleanupVM() error { // Cleanup vm path - dir := filepath.Join(q.store.RunVMStoragePath(), q.id) + dir := filepath.Join(q.config.VMStorePath, q.id) // If it's a symlink, remove both dir and the target. // This can happen when vm template links a sandbox to a vm. @@ -1023,7 +1015,7 @@ func (q *qemu) cleanupVM() error { } if q.config.VMid != "" { - dir = filepath.Join(q.store.RunStoragePath(), q.config.VMid) + dir = filepath.Join(q.config.RunStorePath, q.config.VMid) if err := os.RemoveAll(dir); err != nil { q.Logger().WithError(err).WithField("path", dir).Warnf("failed to remove vm path") } @@ -1149,7 +1141,7 @@ func (q *qemu) dumpSandboxMetaInfo(dumpSavePath string) { dumpStatePath := filepath.Join(dumpSavePath, "state") // copy state from /run/vc/sbs to memory dump directory - statePath := filepath.Join(q.store.RunStoragePath(), q.id) + statePath := filepath.Join(q.config.RunStorePath, q.id) command := []string{"/bin/cp", "-ar", statePath, dumpStatePath} q.Logger().WithField("command", command).Info("try to Save sandbox state") if output, err := pkgUtils.RunCommandFull(command, true); err != nil { @@ -1822,7 +1814,7 @@ func (q *qemu) hotplugAddCPUs(amount uint32) (uint32, error) { } // a new vCPU was added, update list of hotplugged vCPUs and Check if all vCPUs were added - q.state.HotpluggedVCPUs = append(q.state.HotpluggedVCPUs, CPUDevice{cpuID}) + q.state.HotpluggedVCPUs = append(q.state.HotpluggedVCPUs, hv.CPUDevice{ID: cpuID}) hotpluggedVCPUs++ if hotpluggedVCPUs == amount { // All vCPUs were hotplugged @@ -2030,7 +2022,7 @@ func (q *qemu) GetVMConsole(ctx context.Context, id string) (string, string, err span, _ := katatrace.Trace(ctx, q.Logger(), "GetVMConsole", qemuTracingTags, map[string]string{"sandbox_id": q.id}) defer span.End() - consoleURL, err := utils.BuildSocketPath(q.store.RunVMStoragePath(), id, consoleSocket) + consoleURL, err := utils.BuildSocketPath(q.config.VMStorePath, id, consoleSocket) if err != nil { return consoleProtoUnix, "", err } @@ -2469,7 +2461,7 @@ func (q *qemu) toGrpc(ctx context.Context) ([]byte, error) { return json.Marshal(&qp) } -func (q *qemu) Save() (s persistapi.HypervisorState) { +func (q *qemu) Save() (s hv.HypervisorState) { // If QEMU isn't even running, there isn't any state to Save if q.stopped { @@ -2488,7 +2480,7 @@ func (q *qemu) Save() (s persistapi.HypervisorState) { s.PCIeRootPort = q.state.PCIeRootPort for _, bridge := range q.arch.getBridges() { - s.Bridges = append(s.Bridges, persistapi.Bridge{ + s.Bridges = append(s.Bridges, hv.Bridge{ DeviceAddr: bridge.Devices, Type: string(bridge.Type), ID: bridge.ID, @@ -2497,14 +2489,14 @@ func (q *qemu) Save() (s persistapi.HypervisorState) { } for _, cpu := range q.state.HotpluggedVCPUs { - s.HotpluggedVCPUs = append(s.HotpluggedVCPUs, persistapi.CPUDevice{ + s.HotpluggedVCPUs = append(s.HotpluggedVCPUs, hv.CPUDevice{ ID: cpu.ID, }) } return } -func (q *qemu) Load(s persistapi.HypervisorState) { +func (q *qemu) Load(s hv.HypervisorState) { q.state.UUID = s.UUID q.state.HotpluggedMemory = s.HotpluggedMemory q.state.HotplugVFIOOnRootBus = s.HotplugVFIOOnRootBus @@ -2516,7 +2508,7 @@ func (q *qemu) Load(s persistapi.HypervisorState) { } for _, cpu := range s.HotpluggedVCPUs { - q.state.HotpluggedVCPUs = append(q.state.HotpluggedVCPUs, CPUDevice{ + q.state.HotpluggedVCPUs = append(q.state.HotpluggedVCPUs, hv.CPUDevice{ ID: cpu.ID, }) } @@ -2543,7 +2535,7 @@ func (q *qemu) Check() error { } func (q *qemu) GenerateSocket(id string) (interface{}, error) { - return generateVMSocket(id, q.store.RunVMStoragePath()) + return generateVMSocket(id, q.config.VMStorePath) } func (q *qemu) IsRateLimiterBuiltin() bool { diff --git a/src/runtime/virtcontainers/qemu_test.go b/src/runtime/virtcontainers/qemu_test.go index deeb482dc6..9104a68923 100644 --- a/src/runtime/virtcontainers/qemu_test.go +++ b/src/runtime/virtcontainers/qemu_test.go @@ -78,7 +78,10 @@ func TestQemuCreateVM(t *testing.T) { store, err := persist.GetDriver() assert.NoError(err) q := &qemu{ - store: store, + config: HypervisorConfig{ + VMStorePath: store.RunVMStoragePath(), + RunStorePath: store.RunStoragePath(), + }, } sandbox := &Sandbox{ ctx: context.Background(), @@ -94,7 +97,7 @@ func TestQemuCreateVM(t *testing.T) { assert.NoError(err) // Create parent dir path for hypervisor.json - parentDir := filepath.Join(q.store.RunStoragePath(), sandbox.id) + parentDir := filepath.Join(store.RunStoragePath(), sandbox.id) assert.NoError(os.MkdirAll(parentDir, DirMode)) err = q.CreateVM(context.Background(), sandbox.id, NetworkNamespace{}, &sandbox.config.HypervisorConfig) @@ -110,7 +113,10 @@ func TestQemuCreateVMMissingParentDirFail(t *testing.T) { store, err := persist.GetDriver() assert.NoError(err) q := &qemu{ - store: store, + config: HypervisorConfig{ + VMStorePath: store.RunVMStoragePath(), + RunStorePath: store.RunStoragePath(), + }, } sandbox := &Sandbox{ ctx: context.Background(), @@ -126,7 +132,7 @@ func TestQemuCreateVMMissingParentDirFail(t *testing.T) { assert.NoError(err) // Ensure parent dir path for hypervisor.json does not exist. - parentDir := filepath.Join(q.store.RunStoragePath(), sandbox.id) + parentDir := filepath.Join(store.RunStoragePath(), sandbox.id) assert.NoError(os.RemoveAll(parentDir)) err = q.CreateVM(context.Background(), sandbox.id, NetworkNamespace{}, &sandbox.config.HypervisorConfig) @@ -192,7 +198,10 @@ func TestQemuKnobs(t *testing.T) { assert.NoError(err) q := &qemu{ - store: sandbox.store, + config: HypervisorConfig{ + VMStorePath: sandbox.store.RunVMStoragePath(), + RunStorePath: sandbox.store.RunStoragePath(), + }, } err = q.CreateVM(context.Background(), sandbox.id, NetworkNamespace{}, &sandbox.config.HypervisorConfig) assert.NoError(err) @@ -325,11 +334,14 @@ func TestQemuGetSandboxConsole(t *testing.T) { store, err := persist.GetDriver() assert.NoError(err) q := &qemu{ - ctx: context.Background(), - store: store, + ctx: context.Background(), + config: HypervisorConfig{ + VMStorePath: store.RunVMStoragePath(), + RunStorePath: store.RunStoragePath(), + }, } sandboxID := "testSandboxID" - expected := filepath.Join(q.store.RunVMStoragePath(), sandboxID, consoleSocket) + expected := filepath.Join(store.RunVMStoragePath(), sandboxID, consoleSocket) proto, result, err := q.GetVMConsole(q.ctx, sandboxID) assert.NoError(err) @@ -460,7 +472,10 @@ func TestQemuFileBackedMem(t *testing.T) { assert.NoError(err) q := &qemu{ - store: sandbox.store, + config: HypervisorConfig{ + VMStorePath: sandbox.store.RunVMStoragePath(), + RunStorePath: sandbox.store.RunStoragePath(), + }, } sandbox.config.HypervisorConfig.SharedFS = config.VirtioFS err = q.CreateVM(context.Background(), sandbox.id, NetworkNamespace{}, &sandbox.config.HypervisorConfig) @@ -475,7 +490,10 @@ func TestQemuFileBackedMem(t *testing.T) { assert.NoError(err) q = &qemu{ - store: sandbox.store, + config: HypervisorConfig{ + VMStorePath: sandbox.store.RunVMStoragePath(), + RunStorePath: sandbox.store.RunStoragePath(), + }, } sandbox.config.HypervisorConfig.BootToBeTemplate = true sandbox.config.HypervisorConfig.SharedFS = config.VirtioFS @@ -491,7 +509,10 @@ func TestQemuFileBackedMem(t *testing.T) { assert.NoError(err) q = &qemu{ - store: sandbox.store, + config: HypervisorConfig{ + VMStorePath: sandbox.store.RunVMStoragePath(), + RunStorePath: sandbox.store.RunStoragePath(), + }, } sandbox.config.HypervisorConfig.FileBackedMemRootDir = "/tmp/xyzabc" err = q.CreateVM(context.Background(), sandbox.id, NetworkNamespace{}, &sandbox.config.HypervisorConfig) @@ -505,7 +526,10 @@ func TestQemuFileBackedMem(t *testing.T) { assert.NoError(err) q = &qemu{ - store: sandbox.store, + config: HypervisorConfig{ + VMStorePath: sandbox.store.RunVMStoragePath(), + RunStorePath: sandbox.store.RunStoragePath(), + }, } sandbox.config.HypervisorConfig.EnableVhostUserStore = true sandbox.config.HypervisorConfig.HugePages = true @@ -518,7 +542,10 @@ func TestQemuFileBackedMem(t *testing.T) { assert.NoError(err) q = &qemu{ - store: sandbox.store, + config: HypervisorConfig{ + VMStorePath: sandbox.store.RunVMStoragePath(), + RunStorePath: sandbox.store.RunStoragePath(), + }, } sandbox.config.HypervisorConfig.EnableVhostUserStore = true sandbox.config.HypervisorConfig.HugePages = false diff --git a/src/runtime/virtcontainers/sandbox.go b/src/runtime/virtcontainers/sandbox.go index c9e07947bb..207ba1d810 100644 --- a/src/runtime/virtcontainers/sandbox.go +++ b/src/runtime/virtcontainers/sandbox.go @@ -530,7 +530,6 @@ func newSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Factor if s.store, err = persist.GetDriver(); err != nil || s.store == nil { return nil, fmt.Errorf("failed to get fs persist driver: %v", err) } - defer func() { if retErr != nil { s.Logger().WithError(retErr).Error("Create new sandbox failed") @@ -538,6 +537,9 @@ func newSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Factor } }() + sandboxConfig.HypervisorConfig.VMStorePath = s.store.RunVMStoragePath() + sandboxConfig.HypervisorConfig.RunStorePath = s.store.RunStoragePath() + spec := s.GetPatchedOCISpec() if spec != nil && spec.Process.SelinuxLabel != "" { sandboxConfig.HypervisorConfig.SELinuxProcessLabel = spec.Process.SelinuxLabel diff --git a/src/runtime/virtcontainers/virtcontainers_test.go b/src/runtime/virtcontainers/virtcontainers_test.go index 56b27b1c53..5772dd60a9 100644 --- a/src/runtime/virtcontainers/virtcontainers_test.go +++ b/src/runtime/virtcontainers/virtcontainers_test.go @@ -60,7 +60,7 @@ var testHyperstartTtySocket = "" func cleanUp() { os.RemoveAll(fs.MockRunStoragePath()) os.RemoveAll(fs.MockRunVMStoragePath()) - syscall.Unmount(getSharePath(testSandboxID), syscall.MNT_DETACH|UmountNoFollow) + syscall.Unmount(GetSharePath(testSandboxID), syscall.MNT_DETACH|UmountNoFollow) os.RemoveAll(testDir) os.MkdirAll(testDir, DirMode) From 2227c46c25530ce12916bf3d7ac9e6ba46fad003 Mon Sep 17 00:00:00 2001 From: Eric Ernst Date: Thu, 11 Nov 2021 16:46:14 -0800 Subject: [PATCH 5/6] vc: hypervisor: use our own logger This'll end up moving to hypervisors pkg, but let's stop using virtLog, instead introduce hvLogger. Fixes: #2884 Signed-off-by: Eric Ernst --- src/runtime/virtcontainers/api.go | 2 +- src/runtime/virtcontainers/clh.go | 2 +- src/runtime/virtcontainers/hypervisor.go | 20 ++++++++++++++------ src/runtime/virtcontainers/qemu.go | 4 ++-- src/runtime/virtcontainers/qemu_amd64.go | 4 ++-- src/runtime/virtcontainers/qemu_arch_base.go | 2 +- src/runtime/virtcontainers/qemu_arm64.go | 2 +- src/runtime/virtcontainers/qemu_ppc64le.go | 4 ++-- src/runtime/virtcontainers/qemu_s390x.go | 2 +- src/runtime/virtcontainers/virtiofsd.go | 2 +- 10 files changed, 26 insertions(+), 18 deletions(-) diff --git a/src/runtime/virtcontainers/api.go b/src/runtime/virtcontainers/api.go index 67e7608acb..14cfbb1093 100644 --- a/src/runtime/virtcontainers/api.go +++ b/src/runtime/virtcontainers/api.go @@ -35,7 +35,7 @@ var virtLog = logrus.WithField("source", "virtcontainers") func SetLogger(ctx context.Context, logger *logrus.Entry) { fields := virtLog.Data virtLog = logger.WithFields(fields) - + SetHypervisorLogger(virtLog) // TODO: this will move to hypervisors pkg deviceApi.SetLogger(virtLog) compatoci.SetLogger(virtLog) deviceConfig.SetLogger(virtLog) diff --git a/src/runtime/virtcontainers/clh.go b/src/runtime/virtcontainers/clh.go index e4ca0a0061..7fad500f55 100644 --- a/src/runtime/virtcontainers/clh.go +++ b/src/runtime/virtcontainers/clh.go @@ -813,7 +813,7 @@ func (clh *cloudHypervisor) AddDevice(ctx context.Context, devInfo interface{}, //########################################################################### func (clh *cloudHypervisor) Logger() *log.Entry { - return virtLog.WithField("subsystem", "cloudHypervisor") + return hvLogger.WithField("subsystem", "cloudHypervisor") } // Adds all capabilities supported by cloudHypervisor implementation of hypervisor interface diff --git a/src/runtime/virtcontainers/hypervisor.go b/src/runtime/virtcontainers/hypervisor.go index f446a078bf..06a0385c93 100644 --- a/src/runtime/virtcontainers/hypervisor.go +++ b/src/runtime/virtcontainers/hypervisor.go @@ -18,6 +18,8 @@ import ( "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils" + + "github.com/sirupsen/logrus" ) // HypervisorType describes an hypervisor type. @@ -45,14 +47,10 @@ const ( // MockHypervisor is a mock hypervisor for testing purposes MockHypervisor HypervisorType = "mock" -) -const ( procMemInfo = "/proc/meminfo" procCPUInfo = "/proc/cpuinfo" -) -const ( defaultVCPUs = 1 // 2 GiB defaultMemSzMiB = 2048 @@ -73,6 +71,10 @@ const ( MinHypervisorMemory = 256 ) +var ( + hvLogger = logrus.WithField("source", "virtcontainers/hypervisor") +) + // In some architectures the maximum number of vCPUs depends on the number of physical cores. var defaultMaxQemuVCPUs = MaxQemuVCPUs() @@ -143,6 +145,12 @@ type MemoryDevice struct { Probe bool } +// SetHypervisorLogger sets up a logger for the hypervisor part of this pkg +func SetHypervisorLogger(logger *logrus.Entry) { + fields := hvLogger.Data + hvLogger = logger.WithFields(fields) +} + // Set sets an hypervisor type based on the input string. func (hType *HypervisorType) Set(value string) error { switch value { @@ -604,7 +612,7 @@ func (conf *HypervisorConfig) AddCustomAsset(a *types.Asset) error { return fmt.Errorf("Invalid %s at %s", a.Type(), a.Path()) } - virtLog.Debugf("Using custom %v asset %s", a.Type(), a.Path()) + hvLogger.Debugf("Using custom %v asset %s", a.Type(), a.Path()) if conf.customAssets == nil { conf.customAssets = make(map[types.AssetType]*types.Asset) @@ -872,7 +880,7 @@ func RunningOnVMM(cpuInfoPath string) (bool, error) { return flags["hypervisor"], nil } - virtLog.WithField("arch", runtime.GOARCH).Info("Unable to know if the system is running inside a VM") + hvLogger.WithField("arch", runtime.GOARCH).Info("Unable to know if the system is running inside a VM") return false, nil } diff --git a/src/runtime/virtcontainers/qemu.go b/src/runtime/virtcontainers/qemu.go index 1c1073b24f..d0b3bd5c5f 100644 --- a/src/runtime/virtcontainers/qemu.go +++ b/src/runtime/virtcontainers/qemu.go @@ -141,7 +141,7 @@ type qmpLogger struct { func newQMPLogger() qmpLogger { return qmpLogger{ - logger: virtLog.WithField("subsystem", "qmp"), + logger: hvLogger.WithField("subsystem", "qmp"), } } @@ -163,7 +163,7 @@ func (l qmpLogger) Errorf(format string, v ...interface{}) { // Logger returns a logrus logger appropriate for logging qemu messages func (q *qemu) Logger() *logrus.Entry { - return virtLog.WithField("subsystem", "qemu") + return hvLogger.WithField("subsystem", "qemu") } func (q *qemu) kernelParameters() string { diff --git a/src/runtime/virtcontainers/qemu_amd64.go b/src/runtime/virtcontainers/qemu_amd64.go index 689575ab5d..c1464809a4 100644 --- a/src/runtime/virtcontainers/qemu_amd64.go +++ b/src/runtime/virtcontainers/qemu_amd64.go @@ -169,7 +169,7 @@ func (q *qemuAmd64) cpuModel() string { // VMX is not migratable yet. // issue: https://github.com/kata-containers/runtime/issues/1750 if q.vmFactory { - virtLog.WithField("subsystem", "qemuAmd64").Warn("VMX is not migratable yet: turning it off") + hvLogger.WithField("subsystem", "qemuAmd64").Warn("VMX is not migratable yet: turning it off") cpuModel += ",vmx=off" } @@ -200,7 +200,7 @@ func (q *qemuAmd64) enableProtection() error { if err != nil { return err } - logger := virtLog.WithFields(logrus.Fields{ + logger := hvLogger.WithFields(logrus.Fields{ "subsystem": "qemuAmd64", "machine": q.qemuMachine, "kernel-params-debug": q.kernelParamsDebug, diff --git a/src/runtime/virtcontainers/qemu_arch_base.go b/src/runtime/virtcontainers/qemu_arch_base.go index 426da6bac6..97cd6eb830 100644 --- a/src/runtime/virtcontainers/qemu_arch_base.go +++ b/src/runtime/virtcontainers/qemu_arch_base.go @@ -846,6 +846,6 @@ func (q *qemuArchBase) setPFlash(p []string) { // append protection device func (q *qemuArchBase) appendProtectionDevice(devices []govmmQemu.Device, firmware string) ([]govmmQemu.Device, string, error) { - virtLog.WithField("arch", runtime.GOARCH).Warnf("Confidential Computing has not been implemented for this architecture") + hvLogger.WithField("arch", runtime.GOARCH).Warnf("Confidential Computing has not been implemented for this architecture") return devices, firmware, nil } diff --git a/src/runtime/virtcontainers/qemu_arm64.go b/src/runtime/virtcontainers/qemu_arm64.go index 2cd869a8c3..452493ce16 100644 --- a/src/runtime/virtcontainers/qemu_arm64.go +++ b/src/runtime/virtcontainers/qemu_arm64.go @@ -171,6 +171,6 @@ func (q *qemuArm64) enableProtection() error { func (q *qemuArm64) appendProtectionDevice(devices []govmmQemu.Device, firmware string) ([]govmmQemu.Device, string, error) { err := q.enableProtection() - virtLog.WithField("arch", runtime.GOARCH).Warnf("%v", err) + hvLogger.WithField("arch", runtime.GOARCH).Warnf("%v", err) return devices, firmware, err } diff --git a/src/runtime/virtcontainers/qemu_ppc64le.go b/src/runtime/virtcontainers/qemu_ppc64le.go index 51c9003ba2..00fec3529e 100644 --- a/src/runtime/virtcontainers/qemu_ppc64le.go +++ b/src/runtime/virtcontainers/qemu_ppc64le.go @@ -51,7 +51,7 @@ var supportedQemuMachine = govmmQemu.Machine{ // Logger returns a logrus logger appropriate for logging qemu messages func (q *qemuPPC64le) Logger() *logrus.Entry { - return virtLog.WithField("subsystem", "qemuPPC64le") + return hvLogger.WithField("subsystem", "qemuPPC64le") } // MaxQemuVCPUs returns the maximum number of vCPUs supported @@ -141,7 +141,7 @@ func (q *qemuPPC64le) enableProtection() error { q.qemuMachine.Options += "," } q.qemuMachine.Options += fmt.Sprintf("confidential-guest-support=%s", pefID) - virtLog.WithFields(logrus.Fields{ + hvLogger.WithFields(logrus.Fields{ "subsystem": "qemuPPC64le", "machine": q.qemuMachine, "kernel-params": q.kernelParams, diff --git a/src/runtime/virtcontainers/qemu_s390x.go b/src/runtime/virtcontainers/qemu_s390x.go index b7611deccd..d6c013156c 100644 --- a/src/runtime/virtcontainers/qemu_s390x.go +++ b/src/runtime/virtcontainers/qemu_s390x.go @@ -324,7 +324,7 @@ func (q *qemuS390x) enableProtection() error { q.qemuMachine.Options += "," } q.qemuMachine.Options += fmt.Sprintf("confidential-guest-support=%s", secExecID) - virtLog.WithFields(logrus.Fields{ + hvLogger.WithFields(logrus.Fields{ "subsystem": logSubsystem, "machine": q.qemuMachine}). Info("Enabling guest protection with Secure Execution") diff --git a/src/runtime/virtcontainers/virtiofsd.go b/src/runtime/virtcontainers/virtiofsd.go index d35399bead..baaec862f8 100644 --- a/src/runtime/virtcontainers/virtiofsd.go +++ b/src/runtime/virtcontainers/virtiofsd.go @@ -216,7 +216,7 @@ func (v *virtiofsd) valid() error { } func (v *virtiofsd) Logger() *log.Entry { - return virtLog.WithField("subsystem", "virtiofsd") + return hvLogger.WithField("subsystem", "virtiofsd") } func (v *virtiofsd) kill(ctx context.Context) (err error) { From ce92cadc7d491226cd502d86982e50b7a404613f Mon Sep 17 00:00:00 2001 From: Eric Ernst Date: Thu, 11 Nov 2021 17:50:13 -0800 Subject: [PATCH 6/6] vc: hypervisor: remove setSandbox The hypervisor interface implementation should not know a thing about sandboxes. Fixes: #2882 Signed-off-by: Eric Ernst --- src/runtime/virtcontainers/clh.go | 3 --- src/runtime/virtcontainers/fc.go | 3 --- src/runtime/virtcontainers/hypervisor.go | 2 -- src/runtime/virtcontainers/mock_hypervisor.go | 3 --- src/runtime/virtcontainers/qemu.go | 3 --- src/runtime/virtcontainers/sandbox.go | 2 -- 6 files changed, 16 deletions(-) diff --git a/src/runtime/virtcontainers/clh.go b/src/runtime/virtcontainers/clh.go index 7fad500f55..a3cdf02f6f 100644 --- a/src/runtime/virtcontainers/clh.go +++ b/src/runtime/virtcontainers/clh.go @@ -1271,6 +1271,3 @@ func (clh *cloudHypervisor) vmInfo() (chclient.VmInfo, error) { func (clh *cloudHypervisor) IsRateLimiterBuiltin() bool { return false } - -func (clh *cloudHypervisor) setSandbox(sandbox *Sandbox) { -} diff --git a/src/runtime/virtcontainers/fc.go b/src/runtime/virtcontainers/fc.go index fa4e56ffab..c6dffb337e 100644 --- a/src/runtime/virtcontainers/fc.go +++ b/src/runtime/virtcontainers/fc.go @@ -1274,6 +1274,3 @@ func revertBytes(num uint64) uint64 { } return 1024*revertBytes(a) + b } - -func (fc *firecracker) setSandbox(sandbox *Sandbox) { -} diff --git a/src/runtime/virtcontainers/hypervisor.go b/src/runtime/virtcontainers/hypervisor.go index 06a0385c93..715b4793c5 100644 --- a/src/runtime/virtcontainers/hypervisor.go +++ b/src/runtime/virtcontainers/hypervisor.go @@ -945,6 +945,4 @@ type Hypervisor interface { // check if hypervisor supports built-in rate limiter. IsRateLimiterBuiltin() bool - - setSandbox(sandbox *Sandbox) } diff --git a/src/runtime/virtcontainers/mock_hypervisor.go b/src/runtime/virtcontainers/mock_hypervisor.go index dc4b11ad67..111707fd9e 100644 --- a/src/runtime/virtcontainers/mock_hypervisor.go +++ b/src/runtime/virtcontainers/mock_hypervisor.go @@ -149,6 +149,3 @@ func (m *mockHypervisor) GenerateSocket(id string) (interface{}, error) { func (m *mockHypervisor) IsRateLimiterBuiltin() bool { return false } - -func (m *mockHypervisor) setSandbox(sandbox *Sandbox) { -} diff --git a/src/runtime/virtcontainers/qemu.go b/src/runtime/virtcontainers/qemu.go index d0b3bd5c5f..9b78fb24ee 100644 --- a/src/runtime/virtcontainers/qemu.go +++ b/src/runtime/virtcontainers/qemu.go @@ -2541,6 +2541,3 @@ func (q *qemu) GenerateSocket(id string) (interface{}, error) { func (q *qemu) IsRateLimiterBuiltin() bool { return false } - -func (q *qemu) setSandbox(sandbox *Sandbox) { -} diff --git a/src/runtime/virtcontainers/sandbox.go b/src/runtime/virtcontainers/sandbox.go index 207ba1d810..6f3956216f 100644 --- a/src/runtime/virtcontainers/sandbox.go +++ b/src/runtime/virtcontainers/sandbox.go @@ -525,8 +525,6 @@ func newSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Factor swapDevices: []*config.BlockDrive{}, } - hypervisor.setSandbox(s) - if s.store, err = persist.GetDriver(); err != nil || s.store == nil { return nil, fmt.Errorf("failed to get fs persist driver: %v", err) }