github/kata-containers
1
0
Fork 1
mirror of https://github.com/kata-containers/kata-containers.git synced 2025-07-31 23:36:12 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #2179 from WeiZhang555/persist-storage

Browse Source 
Support "Configuration" in "newstore" feature
This commit is contained in:
Fupan Li 2019-11-20 10:08:13 +08:00 committed by GitHub
commit 48c8d669fe
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 396 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
cli/main.go
View File

@ -21,6 +21,7 @@ import (
"github.com/kata-containers/runtime/pkg/rootless"
"github.com/kata-containers/runtime/pkg/signals"
vc "github.com/kata-containers/runtime/virtcontainers"
exp "github.com/kata-containers/runtime/virtcontainers/experimental"
vf "github.com/kata-containers/runtime/virtcontainers/factory"
"github.com/kata-containers/runtime/virtcontainers/pkg/oci"
specs "github.com/opencontainers/runtime-spec/specs-go"
@ -346,6 +347,11 @@ func beforeSubcommands(c *cli.Context) error {
"arguments": `"` + args + `"`,
}
err = addExpFeatures(c, runtimeConfig)
if err != nil {
return err
}
kataLog.WithFields(fields).Info()
// make the data accessible to the sub-commands.
@ -401,6 +407,24 @@ func setupTracing(context *cli.Context, rootSpanName string) error {
return nil
}
// add supported experimental features in context
func addExpFeatures(clictx *cli.Context, runtimeConfig oci.RuntimeConfig) error {
ctx, err := cliContextToContext(clictx)
if err != nil {
return err
}
var exps []string
for _, e := range runtimeConfig.Experimental {
exps = append(exps, e.Name)
}
ctx = exp.ContextWithExp(ctx, exps)
// Add tracer to metadata and update the context
clictx.App.Metadata["context"] = ctx
return nil
}
func afterSubcommands(c *cli.Context) error {
ctx, err := cliContextToContext(c)
if err != nil {

10
virtcontainers/api.go
View File

@ -13,6 +13,7 @@ import (
deviceApi "github.com/kata-containers/runtime/virtcontainers/device/api"
deviceConfig "github.com/kata-containers/runtime/virtcontainers/device/config"
"github.com/kata-containers/runtime/virtcontainers/persist/fs"
vcTypes "github.com/kata-containers/runtime/virtcontainers/pkg/types"
"github.com/kata-containers/runtime/virtcontainers/store"
"github.com/kata-containers/runtime/virtcontainers/types"
@ -307,7 +308,14 @@ func ListSandbox(ctx context.Context) ([]SandboxStatus, error) {
span, ctx := trace(ctx, "ListSandbox")
defer span.Finish()
dir, err := os.Open(store.ConfigStoragePath())
var sbsdir string
if supportNewStore(ctx) {
sbsdir = fs.RunStoragePath()
} else {
sbsdir = store.RunStoragePath()
}
dir, err := os.Open(sbsdir)
if err != nil {
if os.IsNotExist(err) {
// No sandbox directory is not an error

1
virtcontainers/container.go
View File

@ -417,6 +417,7 @@ func (c *Container) storeContainer() error {
if err := c.sandbox.Save(); err != nil {
return err
}
return nil
}
return c.store.Store(store.Configuration, *(c.config))
}

17
virtcontainers/experimental/experimental.go
View File

@ -6,6 +6,7 @@
package experimental
import (
"context"
"fmt"
"regexp"
)
@ -22,8 +23,11 @@ type Feature struct {
ExpRelease string
}
type contextKey struct{}
var (
supportedFeatures = make(map[string]Feature)
expContextKey = contextKey{}
)
// Register register a new experimental feature
@ -61,3 +65,16 @@ func validateFeature(feature Feature) error {
return nil
}
func ContextWithExp(ctx context.Context, names []string) context.Context {
return context.WithValue(ctx, expContextKey, names)
}
func ExpFromContext(ctx context.Context) []string {
value := ctx.Value(expContextKey)
if value == nil {
return nil
}
names := value.([]string)
return names
}

231
virtcontainers/persist.go
View File

@ -13,6 +13,7 @@ import (
"github.com/kata-containers/runtime/virtcontainers/persist"
persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api"
"github.com/kata-containers/runtime/virtcontainers/types"
"github.com/mitchellh/mapstructure"
)
var (
@ -171,6 +172,119 @@ func (s *Sandbox) dumpNetwork(ss *persistapi.SandboxState) {
}
}
func (s *Sandbox) dumpConfig(ss *persistapi.SandboxState) {
sconfig := s.config
ss.Config = persistapi.SandboxConfig{
HypervisorType: string(sconfig.HypervisorType),
AgentType: string(sconfig.AgentType),
ProxyType: string(sconfig.ProxyType),
ProxyConfig: persistapi.ProxyConfig{
Path: sconfig.ProxyConfig.Path,
Debug: sconfig.ProxyConfig.Debug,
},
ShimType: string(sconfig.ShimType),
NetworkConfig: persistapi.NetworkConfig{
NetNSPath: sconfig.NetworkConfig.NetNSPath,
NetNsCreated: sconfig.NetworkConfig.NetNsCreated,
DisableNewNetNs: sconfig.NetworkConfig.DisableNewNetNs,
InterworkingModel: int(sconfig.NetworkConfig.InterworkingModel),
},
ShmSize: sconfig.ShmSize,
SharePidNs: sconfig.SharePidNs,
Stateful: sconfig.Stateful,
SystemdCgroup: sconfig.SystemdCgroup,
SandboxCgroupOnly: sconfig.SandboxCgroupOnly,
DisableGuestSeccomp: sconfig.DisableGuestSeccomp,
}
for _, e := range sconfig.Experimental {
ss.Config.Experimental = append(ss.Config.Experimental, e.Name)
}
ss.Config.HypervisorConfig = persistapi.HypervisorConfig{
NumVCPUs: sconfig.HypervisorConfig.NumVCPUs,
DefaultMaxVCPUs: sconfig.HypervisorConfig.DefaultMaxVCPUs,
MemorySize: sconfig.HypervisorConfig.MemorySize,
DefaultBridges: sconfig.HypervisorConfig.DefaultBridges,
Msize9p: sconfig.HypervisorConfig.Msize9p,
MemSlots: sconfig.HypervisorConfig.MemSlots,
MemOffset: sconfig.HypervisorConfig.MemOffset,
VirtioFSCacheSize: sconfig.HypervisorConfig.VirtioFSCacheSize,
KernelPath: sconfig.HypervisorConfig.KernelPath,
ImagePath: sconfig.HypervisorConfig.ImagePath,
InitrdPath: sconfig.HypervisorConfig.InitrdPath,
FirmwarePath: sconfig.HypervisorConfig.FirmwarePath,
MachineAccelerators: sconfig.HypervisorConfig.MachineAccelerators,
HypervisorPath: sconfig.HypervisorConfig.HypervisorPath,
HypervisorCtlPath: sconfig.HypervisorConfig.HypervisorCtlPath,
JailerPath: sconfig.HypervisorConfig.JailerPath,
BlockDeviceDriver: sconfig.HypervisorConfig.BlockDeviceDriver,
HypervisorMachineType: sconfig.HypervisorConfig.HypervisorMachineType,
MemoryPath: sconfig.HypervisorConfig.MemoryPath,
DevicesStatePath: sconfig.HypervisorConfig.DevicesStatePath,
EntropySource: sconfig.HypervisorConfig.EntropySource,
SharedFS: sconfig.HypervisorConfig.SharedFS,
VirtioFSDaemon: sconfig.HypervisorConfig.VirtioFSDaemon,
VirtioFSCache: sconfig.HypervisorConfig.VirtioFSCache,
VirtioFSExtraArgs: sconfig.HypervisorConfig.VirtioFSExtraArgs[:],
BlockDeviceCacheSet: sconfig.HypervisorConfig.BlockDeviceCacheSet,
BlockDeviceCacheDirect: sconfig.HypervisorConfig.BlockDeviceCacheDirect,
BlockDeviceCacheNoflush: sconfig.HypervisorConfig.BlockDeviceCacheNoflush,
DisableBlockDeviceUse: sconfig.HypervisorConfig.DisableBlockDeviceUse,
EnableIOThreads: sconfig.HypervisorConfig.EnableIOThreads,
Debug: sconfig.HypervisorConfig.Debug,
MemPrealloc: sconfig.HypervisorConfig.MemPrealloc,
HugePages: sconfig.HypervisorConfig.HugePages,
FileBackedMemRootDir: sconfig.HypervisorConfig.FileBackedMemRootDir,
Realtime: sconfig.HypervisorConfig.Realtime,
Mlock: sconfig.HypervisorConfig.Mlock,
DisableNestingChecks: sconfig.HypervisorConfig.DisableNestingChecks,
UseVSock: sconfig.HypervisorConfig.UseVSock,
HotplugVFIOOnRootBus: sconfig.HypervisorConfig.HotplugVFIOOnRootBus,
BootToBeTemplate: sconfig.HypervisorConfig.BootToBeTemplate,
BootFromTemplate: sconfig.HypervisorConfig.BootFromTemplate,
DisableVhostNet: sconfig.HypervisorConfig.DisableVhostNet,
GuestHookPath: sconfig.HypervisorConfig.GuestHookPath,
VMid: sconfig.HypervisorConfig.VMid,
}
if sconfig.AgentType == "kata" {
var sagent KataAgentConfig
err := mapstructure.Decode(sconfig.AgentConfig, &sagent)
if err != nil {
s.Logger().WithError(err).Error("internal error: KataAgentConfig failed to decode")
} else {
ss.Config.KataAgentConfig = &persistapi.KataAgentConfig{
LongLiveConn: sagent.LongLiveConn,
UseVSock: sagent.UseVSock,
}
}
}
if sconfig.ShimType == "kataShim" {
var shim ShimConfig
err := mapstructure.Decode(sconfig.ShimConfig, &shim)
if err != nil {
s.Logger().WithError(err).Error("internal error: ShimConfig failed to decode")
} else {
ss.Config.KataShimConfig = &persistapi.ShimConfig{
Path: shim.Path,
Debug: shim.Debug,
}
}
}
for _, contConf := range sconfig.Containers {
ss.Config.ContainerConfigs = append(ss.Config.ContainerConfigs, persistapi.ContainerConfig{
ID: contConf.ID,
Annotations: contConf.Annotations,
RootFs: contConf.RootFs.Target,
Resources: contConf.Resources,
})
}
}
func (s *Sandbox) Save() error {
var (
ss = persistapi.SandboxState{}
@ -185,6 +299,7 @@ func (s *Sandbox) Save() error {
s.dumpMounts(cs)
s.dumpAgent(&ss)
s.dumpNetwork(&ss)
s.dumpConfig(&ss)
if err := s.newStore.ToDisk(ss, cs); err != nil {
return err
@ -335,3 +450,119 @@ func (s *Sandbox) supportNewStore() bool {
}
return false
}
func loadSandboxConfig(id string) (*SandboxConfig, error) {
store, err := persist.GetDriver("fs")
if err != nil || store == nil {
return nil, errors.New("failed to get fs persist driver")
}
ss, _, err := store.FromDisk(id)
if err != nil {
return nil, err
}
savedConf := ss.Config
sconfig := &SandboxConfig{
ID: id,
HypervisorType: HypervisorType(savedConf.HypervisorType),
AgentType: AgentType(savedConf.AgentType),
ProxyType: ProxyType(savedConf.ProxyType),
ProxyConfig: ProxyConfig{
Path: savedConf.ProxyConfig.Path,
Debug: savedConf.ProxyConfig.Debug,
},
ShimType: ShimType(savedConf.ShimType),
NetworkConfig: NetworkConfig{
NetNSPath: savedConf.NetworkConfig.NetNSPath,
NetNsCreated: savedConf.NetworkConfig.NetNsCreated,
DisableNewNetNs: savedConf.NetworkConfig.DisableNewNetNs,
InterworkingModel: NetInterworkingModel(savedConf.NetworkConfig.InterworkingModel),
},
ShmSize: savedConf.ShmSize,
SharePidNs: savedConf.SharePidNs,
Stateful: savedConf.Stateful,
SystemdCgroup: savedConf.SystemdCgroup,
SandboxCgroupOnly: savedConf.SandboxCgroupOnly,
DisableGuestSeccomp: savedConf.DisableGuestSeccomp,
}
for _, name := range savedConf.Experimental {
sconfig.Experimental = append(sconfig.Experimental, *exp.Get(name))
}
hconf := savedConf.HypervisorConfig
sconfig.HypervisorConfig = HypervisorConfig{
NumVCPUs: hconf.NumVCPUs,
DefaultMaxVCPUs: hconf.DefaultMaxVCPUs,
MemorySize: hconf.MemorySize,
DefaultBridges: hconf.DefaultBridges,
Msize9p: hconf.Msize9p,
MemSlots: hconf.MemSlots,
MemOffset: hconf.MemOffset,
VirtioFSCacheSize: hconf.VirtioFSCacheSize,
KernelPath: hconf.KernelPath,
ImagePath: hconf.ImagePath,
InitrdPath: hconf.InitrdPath,
FirmwarePath: hconf.FirmwarePath,
MachineAccelerators: hconf.MachineAccelerators,
HypervisorPath: hconf.HypervisorPath,
HypervisorCtlPath: hconf.HypervisorCtlPath,
JailerPath: hconf.JailerPath,
BlockDeviceDriver: hconf.BlockDeviceDriver,
HypervisorMachineType: hconf.HypervisorMachineType,
MemoryPath: hconf.MemoryPath,
DevicesStatePath: hconf.DevicesStatePath,
EntropySource: hconf.EntropySource,
SharedFS: hconf.SharedFS,
VirtioFSDaemon: hconf.VirtioFSDaemon,
VirtioFSCache: hconf.VirtioFSCache,
VirtioFSExtraArgs: hconf.VirtioFSExtraArgs[:],
BlockDeviceCacheSet: hconf.BlockDeviceCacheSet,
BlockDeviceCacheDirect: hconf.BlockDeviceCacheDirect,
BlockDeviceCacheNoflush: hconf.BlockDeviceCacheNoflush,
DisableBlockDeviceUse: hconf.DisableBlockDeviceUse,
EnableIOThreads: hconf.EnableIOThreads,
Debug: hconf.Debug,
MemPrealloc: hconf.MemPrealloc,
HugePages: hconf.HugePages,
FileBackedMemRootDir: hconf.FileBackedMemRootDir,
Realtime: hconf.Realtime,
Mlock: hconf.Mlock,
DisableNestingChecks: hconf.DisableNestingChecks,
UseVSock: hconf.UseVSock,
HotplugVFIOOnRootBus: hconf.HotplugVFIOOnRootBus,
BootToBeTemplate: hconf.BootToBeTemplate,
BootFromTemplate: hconf.BootFromTemplate,
DisableVhostNet: hconf.DisableVhostNet,
GuestHookPath: hconf.GuestHookPath,
VMid: hconf.VMid,
}
if savedConf.AgentType == "kata" {
sconfig.AgentConfig = KataAgentConfig{
LongLiveConn: savedConf.KataAgentConfig.LongLiveConn,
UseVSock: savedConf.KataAgentConfig.UseVSock,
}
}
if savedConf.ShimType == "kataShim" {
sconfig.ShimConfig = ShimConfig{
Path: savedConf.KataShimConfig.Path,
Debug: savedConf.KataShimConfig.Debug,
}
}
for _, contConf := range savedConf.ContainerConfigs {
sconfig.Containers = append(sconfig.Containers, ContainerConfig{
ID: contConf.ID,
Annotations: contConf.Annotations,
Resources: contConf.Resources,
RootFs: RootFs{
Target: contConf.RootFs,
},
})
}
return sconfig, nil
}

78
virtcontainers/persist/api/config.go
View File

@ -6,17 +6,9 @@
package persistapi
// Param is a key/value representation for hypervisor and kernel parameters.
type Param struct {
Key string
Value string
}
// Asset saves hypervisor asset
type Asset struct {
Path string `json:"path"`
Custom bool `json:"bool"`
}
import (
specs "github.com/opencontainers/runtime-spec/specs-go"
)
// HypervisorConfig saves configurations of sandbox hypervisor
type HypervisorConfig struct {
@ -42,11 +34,8 @@ type HypervisorConfig struct {
// MemOffset specifies memory space for nvdimm device
MemOffset uint32
// KernelParams are additional guest kernel parameters.
KernelParams []Param
// HypervisorParams are additional hypervisor parameters.
HypervisorParams []Param
// VirtioFSCacheSize is the DAX cache size in MiB
VirtioFSCacheSize uint32
// KernelPath is the guest kernel host path.
KernelPath string
@ -67,6 +56,12 @@ type HypervisorConfig struct {
// HypervisorPath is the hypervisor executable host path.
HypervisorPath string
// HypervisorCtlPath is the hypervisor ctl executable host path.
HypervisorCtlPath string
// JailerPath is the jailer executable host path.
JailerPath string
// BlockDeviceDriver specifies the driver to be used for block device
// either VirtioSCSI or VirtioBlock with the default driver being defaultBlockDriver
BlockDeviceDriver string
@ -87,11 +82,19 @@ type HypervisorConfig struct {
// entropy (/dev/random, /dev/urandom or real hardware RNG device)
EntropySource string
// customAssets is a map of assets.
// Each value in that map takes precedence over the configured assets.
// For example, if there is a value for the "kernel" key in this map,
// it will be used for the sandbox's kernel path instead of KernelPath.
CustomAssets map[string]*Asset
// Shared file system type:
// - virtio-9p (default)
// - virtio-fs
SharedFS string
// VirtioFSDaemon is the virtio-fs vhost-user daemon path
VirtioFSDaemon string
// VirtioFSCache cache mode for fs version cache or "none"
VirtioFSCache string
// VirtioFSExtraArgs passes options to virtiofsd daemon
VirtioFSExtraArgs []string
// BlockDeviceCacheSet specifies cache-related options will be set to block devices or not.
BlockDeviceCacheSet bool
@ -154,6 +157,10 @@ type HypervisorConfig struct {
// GuestHookPath is the path within the VM that will be used for 'drop-in' hooks
GuestHookPath string
// VMid is the id of the VM that create the hypervisor if the VM is created by the factory.
// VMid is "" if the hypervisor is not created by the factory.
VMid string
}
// KataAgentConfig is a structure storing information needed
@ -186,6 +193,18 @@ type ShimConfig struct {
// NetworkConfig is the network configuration related to a network.
type NetworkConfig struct {
NetNSPath string
NetNsCreated bool
DisableNewNetNs bool
InterworkingModel int
}
type ContainerConfig struct {
ID string
Annotations map[string]string
RootFs string
// Resources for recoding update
Resources specs.LinuxResources
}
// SandboxConfig is a sandbox configuration.
@ -195,17 +214,15 @@ type SandboxConfig struct {
HypervisorConfig HypervisorConfig
// only one agent config can be non-nil according to agent type
AgentType string
KataAgentConfig *KataAgentConfig `json:",omitempty"`
HyperstartConfig *HyperstartConfig `json:",omitempty"`
AgentType string
KataAgentConfig *KataAgentConfig `json:",omitempty"`
ProxyType string
ProxyConfig ProxyConfig
ShimType string
KataShimConfig ShimConfig
KataShimConfig *ShimConfig
NetworkModel string
NetworkConfig NetworkConfig
ShmSize uint64
@ -220,11 +237,18 @@ type SandboxConfig struct {
// SystemdCgroup enables systemd cgroup support
SystemdCgroup bool
// SandboxCgroupOnly enables cgroup only at podlevel in the host
SandboxCgroupOnly bool
DisableGuestSeccomp bool
// Experimental enables experimental features
Experimental bool
Experimental []string
// Information for fields not saved:
// * Annotation: this is kind of casual data, we don't need casual data in persist file,
// if you know this data needs to persist, please gives it
// a specific field
ContainerConfigs []ContainerConfig
}

19
virtcontainers/persist/fs/fs.go
View File

@ -14,6 +14,7 @@ import (
"path/filepath"
"syscall"
"github.com/kata-containers/runtime/pkg/rootless"
persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api"
"github.com/sirupsen/logrus"
)
@ -36,9 +37,15 @@ const storagePathSuffix = "vc"
// sandboxPathSuffix is the suffix used for sandbox storage
const sandboxPathSuffix = "sbs"
// runStoragePath is the sandbox runtime directory.
// RunStoragePath is the sandbox runtime directory.
// It will contain one state.json and one lock file for each created sandbox.
var runStoragePath = filepath.Join("/run", storagePathSuffix, sandboxPathSuffix)
var RunStoragePath = func() string {
path := filepath.Join("/run", storagePathSuffix, sandboxPathSuffix)
if rootless.IsRootless() {
return filepath.Join(rootless.GetRootlessDir(), path)
}
return path
}
// FS storage driver implementation
type FS struct {
@ -76,7 +83,7 @@ func (fs *FS) sandboxDir() (string, error) {
return "", fmt.Errorf("sandbox container id required")
}
return filepath.Join(runStoragePath, id), nil
return filepath.Join(RunStoragePath(), id), nil
}
// ToDisk sandboxState and containerState to disk
@ -254,8 +261,10 @@ func (fs *FS) unlock() error {
return nil
}
// TestSetRunStoragePath set runStoragePath to path
// TestSetRunStoragePath set RunStoragePath to path
// this function is only used for testing purpose
func TestSetRunStoragePath(path string) {
runStoragePath = path
RunStoragePath = func() string {
return path
}
}

71
virtcontainers/sandbox.go
View File

@ -242,7 +242,10 @@ func (s *Sandbox) SetAnnotations(annotations map[string]string) error {
s.config.Annotations[k] = v
}
return s.store.Store(store.Configuration, *(s.config))
if !s.supportNewStore() {
return s.store.Store(store.Configuration, *(s.config))
}
return nil
}
// GetAnnotations returns sandbox's annotations
@ -617,25 +620,24 @@ func (s *Sandbox) storeSandbox() error {
span, _ := s.trace("storeSandbox")
defer span.Finish()
err := s.store.Store(store.Configuration, *(s.config))
if err != nil {
return err
}
for _, container := range s.containers {
err = container.store.Store(store.Configuration, *(container.config))
if err != nil {
return err
}
}
if s.supportNewStore() {
// flush data to storage
if err := s.Save(); err != nil {
return err
}
}
} else {
err := s.store.Store(store.Configuration, *(s.config))
if err != nil {
return err
}
for _, container := range s.containers {
err = container.store.Store(store.Configuration, *(container.config))
if err != nil {
return err
}
}
}
return nil
}
@ -673,6 +675,22 @@ func unlockSandbox(ctx context.Context, sandboxID, token string) error {
return store.Unlock(token)
}
func supportNewStore(ctx context.Context) bool {
if exp.Get("newstore") == nil {
return false
}
// check if client context enabled "newstore" feature
exps := exp.ExpFromContext(ctx)
for _, v := range exps {
if v == "newstore" {
return true
}
}
return false
}
// fetchSandbox fetches a sandbox config from a sandbox ID and returns a sandbox.
func fetchSandbox(ctx context.Context, sandboxID string) (sandbox *Sandbox, err error) {
virtLog.Info("fetch sandbox")
@ -685,15 +703,24 @@ func fetchSandbox(ctx context.Context, sandboxID string) (sandbox *Sandbox, err
return sandbox, err
}
// We're bootstrapping
vcStore, err := store.NewVCSandboxStore(ctx, sandboxID)
if err != nil {
return nil, err
}
var config SandboxConfig
if err := vcStore.Load(store.Configuration, &config); err != nil {
return nil, err
if supportNewStore(ctx) {
c, err := loadSandboxConfig(sandboxID)
if err != nil {
return nil, err
}
config = *c
} else {
// We're bootstrapping
vcStore, err := store.NewVCSandboxStore(ctx, sandboxID)
if err != nil {
return nil, err
}
if err := vcStore.Load(store.Configuration, &config); err != nil {
return nil, err
}
}
// fetchSandbox is not suppose to create new sandbox VM.